lfss 0.7.3__py3-none-any.whl → 0.7.4__py3-none-any.whl

docs/Permission.md

@@ -18,6 +18,13 @@ Non-admin users can access files based on:
 - If the file is `unset`, then the file's permission is inherited from the owner's permission.
 - If both the owner and the file have `unset` permission, then the file is `public`.
 
+### Meta-data access
+- Non-login users can't access any file-meta.
+- All users can access the file-meta of files under their own path.
+- For files under other users' path, the file-meta is determined in a way same as file access.
+- Admins can access the path-meta of all users.
+- All users can access the path-meta of their own path.
+
 ### Path-listing
 - Non-login users cannot list any files.
 - All users can list the files under their own path 

lfss/cli/cli.py

@@ -1,22 +1,32 @@
-from lfss.client import Connector, upload_directory
+from lfss.client import Connector, upload_directory, upload_file, download_file, download_directory
 from lfss.src.datatype import FileReadPermission
 from pathlib import Path
 import argparse
 
 def parse_arguments():
     parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
 
     sp = parser.add_subparsers(dest="command", required=True)
 
     # upload
     sp_upload = sp.add_parser("upload", help="Upload files")
     sp_upload.add_argument("src", help="Source file or directory", type=str)
-    sp_upload.add_argument("dst", help="Destination path", type=str)
+    sp_upload.add_argument("dst", help="Destination url path", type=str)
     sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
     sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
     sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip", "skip-ahead"], default="abort", help="Conflict resolution")
     sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
-    sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries, only works with directory upload")
+    sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries")
+
+    # download
+    sp_download = sp.add_parser("download", help="Download files")
+    sp_download.add_argument("src", help="Source url path", type=str)
+    sp_download.add_argument("dst", help="Destination file or directory", type=str)
+    sp_download.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent downloads")
+    sp_download.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory download")
+    sp_download.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
+    sp_download.add_argument("--retries", type=int, default=0, help="Number of retries")
 
     return parser.parse_args()
 
@@ -28,9 +38,9 @@ def main():
         if src_path.is_dir():
             failed_upload = upload_directory(
                 connector, args.src, args.dst, 
-                verbose=True, 
+                verbose=args.verbose,
                 n_concurrent=args.jobs, 
-                n_reties=args.retries, 
+                n_retries=args.retries, 
                 interval=args.interval,
                 conflict=args.conflict,
                 permission=args.permission
@@ -40,13 +50,46 @@ def main():
                 for path in failed_upload:
                     print(f"  {path}")
         else:
-            with open(args.src, 'rb') as f:
-                connector.put(
-                    args.dst, 
-                    f.read(), 
-                    conflict=args.conflict,
-                    permission=args.permission
-                    )
+            success = upload_file(
+                connector, 
+                file_path = args.src, 
+                dst_url = args.dst, 
+                verbose=args.verbose,
+                n_retries=args.retries, 
+                interval=args.interval,
+                conflict=args.conflict,
+                permission=args.permission
+            )
+            if not success:
+                print("Failed to upload.")
+    
+    elif args.command == "download":
+        is_dir = args.src.endswith("/")
+        if is_dir:
+            failed_download = download_directory(
+                connector, args.src, args.dst, 
+                verbose=args.verbose,
+                n_concurrent=args.jobs, 
+                n_retries=args.retries, 
+                interval=args.interval,
+                overwrite=args.overwrite
+            )
+            if failed_download:
+                print("Failed to download:")
+                for path in failed_download:
+                    print(f"  {path}")
+        else:
+            success = download_file(
+                connector, 
+                src_url = args.src, 
+                file_path = args.dst, 
+                verbose=args.verbose,
+                n_retries=args.retries, 
+                interval=args.interval,
+                overwrite=args.overwrite
+            )
+            if not success:
+                print("Failed to download.")
     else:
         raise NotImplementedError(f"Command {args.command} not implemented.")
     

lfss/client/__init__.py

@@ -1,14 +1,45 @@
-import os, time
+import os, time, pathlib
 from threading import Lock
 from concurrent.futures import ThreadPoolExecutor
 from .api import Connector
 
+def upload_file(
+    connector: Connector, 
+    file_path: str, 
+    dst_url: str, 
+    n_retries: int = 0, 
+    interval: float = 0, 
+    verbose: bool = False, 
+    **put_kwargs
+    ):
+    this_try = 0
+    while this_try <= n_retries:
+        try:
+            with open(file_path, 'rb') as f:
+                blob = f.read()
+            connector.put(dst_url, blob, **put_kwargs)
+            break
+        except Exception as e:
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+            if verbose:
+                print(f"Error uploading {file_path}: {e}, retrying...")
+            this_try += 1
+        finally:
+            time.sleep(interval)
+
+    if this_try > n_retries:
+        if verbose:
+            print(f"Failed to upload {file_path} after {n_retries} retries.")
+        return False
+    return True
+
 def upload_directory(
     connector: Connector, 
     directory: str, 
     path: str, 
     n_concurrent: int = 1,
-    n_reties: int = 0,
+    n_retries: int = 0,
     interval: float = 0,
     verbose: bool = False,
     **put_kwargs
@@ -16,6 +47,7 @@ def upload_directory(
     assert path.endswith('/'), "Path must end with a slash."
     if path.startswith('/'):
         path = path[1:]
+    directory = str(directory)
     
     _counter = 0
     _counter_lock = Lock()
@@ -30,31 +62,94 @@ def upload_directory(
             if verbose:
                 print(f"[{this_count}] Uploading {file_path} to {dst_path}")
 
-        this_try = 0
-        with open(file_path, 'rb') as f:
-            blob = f.read()
-
-        while this_try <= n_reties:
-            try:
-                connector.put(dst_path, blob, **put_kwargs)
-                break
-            except Exception as e:
-                if isinstance(e, KeyboardInterrupt):
-                    raise e
-                if verbose:
-                    print(f"[{this_count}] Error uploading {file_path}: {e}, retrying...")
-                this_try += 1
-            finally:
-                time.sleep(interval)
-
-        if this_try > n_reties:
+        if not upload_file(
+            connector, file_path, dst_path, 
+            n_retries=n_retries, interval=interval, verbose=verbose, **put_kwargs
+            ):
             faild_files.append(file_path)
-            if verbose:
-                print(f"[{this_count}] Failed to upload {file_path} after {n_reties} retries.")
 
     with ThreadPoolExecutor(n_concurrent) as executor:
         for root, dirs, files in os.walk(directory):
             for file in files:
                 executor.submit(put_file, os.path.join(root, file))
 
-    return faild_files
+    return faild_files
+
+def download_file(
+    connector: Connector, 
+    src_url: str, 
+    file_path: str, 
+    n_retries: int = 0, 
+    interval: float = 0, 
+    verbose: bool = False, 
+    overwrite: bool = False
+    ):
+    this_try = 0
+    while this_try <= n_retries:
+        if not overwrite and os.path.exists(file_path):
+            if verbose:
+                print(f"File {file_path} already exists, skipping download.")
+            return True
+        try:
+            blob = connector.get(src_url)
+            if not blob:
+                return False
+            pathlib.Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+            with open(file_path, 'wb') as f:
+                f.write(blob)
+            break
+        except Exception as e:
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+            if verbose:
+                print(f"Error downloading {src_url}: {e}, retrying...")
+            this_try += 1
+        finally:
+            time.sleep(interval)
+
+    if this_try > n_retries:
+        if verbose:
+            print(f"Failed to download {src_url} after {n_retries} retries.")
+        return False
+    return True
+
+def download_directory(
+    connector: Connector,
+    src_path: str,
+    directory: str,
+    n_concurrent: int = 1,
+    n_retries: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    overwrite: bool = False
+    ) -> list[str]:
+
+    directory = str(directory)
+
+    if not src_path.endswith('/'):
+        src_path += '/'
+    if not directory.endswith(os.sep):
+        directory += os.sep
+    
+    _counter = 0
+    _counter_lock = Lock()
+    failed_files = []
+    def get_file(src_url):
+        nonlocal _counter, failed_files
+        with _counter_lock:
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{directory}{os.path.relpath(src_url, src_path)}"
+            if verbose:
+                print(f"[{this_count}] Downloading {src_url} to {dst_path}")
+
+        if not download_file(
+            connector, src_url, dst_path, 
+            n_retries=n_retries, interval=interval, verbose=verbose, overwrite=overwrite
+            ):
+            failed_files.append(src_url)
+        
+    with ThreadPoolExecutor(n_concurrent) as executor:
+        for file in connector.list_path(src_path).files:
+            executor.submit(get_file, file.url)
+    return failed_files

lfss/client/api.py

@@ -5,6 +5,7 @@ import urllib.parse
 from lfss.src.datatype import (
     FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents
     )
+from lfss.src.utils import ensure_uri_compnents
 
 _default_endpoint = os.environ.get('LFSS_ENDPOINT', 'http://localhost:8000')
 _default_token = os.environ.get('LFSS_TOKEN', '')
@@ -23,6 +24,7 @@ class Connector:
     ):
         if path.startswith('/'):
             path = path[1:]
+        path = ensure_uri_compnents(path)
         def f(**kwargs):
             url = f"{self.config['endpoint']}/{path}" + "?" + urllib.parse.urlencode(search_params)
             headers: dict = kwargs.pop('headers', {})

lfss/src/config.py

@@ -15,6 +15,3 @@ LARGE_BLOB_DIR.mkdir(exist_ok=True)
 LARGE_FILE_BYTES = 8 * 1024 * 1024   # 8MB
 MAX_FILE_BYTES = 512 * 1024 * 1024   # 512MB
 MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB
-
-def hash_credential(username, password):
-    return hashlib.sha256((username + password).encode()).hexdigest()

lfss/src/database.py

@@ -11,9 +11,9 @@ import aiofiles.os
 
 from .connection_pool import execute_sql, unique_cursor, transaction
 from .datatype import UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents
-from .config import LARGE_BLOB_DIR, hash_credential
+from .config import LARGE_BLOB_DIR
 from .log import get_logger
-from .utils import decode_uri_compnents
+from .utils import decode_uri_compnents, hash_credential
 from .error import *
 
 class DBObjectBase(ABC):

lfss/src/server.py

@@ -339,15 +339,34 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
 @router_api.get("/meta")
 @handle_exception
 async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
+    """
+    Permission:
+        for file:
+            if file is under user's path, return the meta, 
+            else, determine by the permission same as get_file
+        for path:
+            if path is under user's path, return the meta, else return 403
+    """
     logger.info(f"GET meta({path}), user: {user.username}")
     path = ensure_uri_compnents(path)
+    is_file = not path.endswith("/")
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
-        get_fn = fconn.get_file_record if not path.endswith("/") else fconn.get_path_record
-        record = await get_fn(path)
-
-    if not record:
-        raise HTTPException(status_code=404, detail="Path not found")
+        if is_file:
+            record = await fconn.get_file_record(path)
+            if not record:
+                raise HTTPException(status_code=404, detail="File not found")
+            if not path.startswith(f"{user.username}/") and not user.is_admin:
+                uconn = UserConn(conn)
+                owner = await uconn.get_user_by_id(record.owner_id)
+                assert owner is not None, "Owner not found"
+                is_allowed, reason = check_user_permission(user, owner, record)
+                if not is_allowed:
+                    raise HTTPException(status_code=403, detail=reason)
+        else:
+            record = await fconn.get_path_record(path)
+            if not path.startswith(f"{user.username}/") and not user.is_admin:
+                raise HTTPException(status_code=403, detail="Permission denied")
     return record
 
 @router_api.post("/meta")

lfss/src/utils.py

@@ -2,6 +2,10 @@ import datetime
 import urllib.parse
 import asyncio
 import functools
+import hashlib
+
+def hash_credential(username: str, password: str):
+    return hashlib.sha256((username + password).encode()).hexdigest()
 
 def encode_uri_compnents(path: str):
     """

{lfss-0.7.3.dist-info → lfss-0.7.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.3
+Version: 0.7.4
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.7.3.dist-info → lfss-0.7.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
 Readme.md,sha256=vsPotlwPAaHI5plh4aaszpi3rr7ZGDn7-wLdEYTWQ0k,1275
 docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
-docs/Permission.md,sha256=e0ZcogBpTxHsKKOUVmNB460-P6AALQToU2ePq-RHHSQ,1930
+docs/Permission.md,sha256=X0VNfBKU52f93QYqcVyiBFJ3yURiSkhIo9S_5fdSgzM,2265
 frontend/api.js,sha256=-ouhsmucEunAK3m1H__MqffQkXAjoeVEfM15BvqfIZs,7677
 frontend/index.html,sha256=VPJDs2LG8ep9kjlsKzjWzpN9vc1VGgdvOUlNTZWyQoQ,2088
 frontend/popup.css,sha256=VzkjG1ZTLxhHMtTyobnlvqYmVsTmdbJJed2Pu1cc06c,1007
@@ -9,25 +9,25 @@ frontend/scripts.js,sha256=hQ8m3L7P-LplLqrPUWD6pBo4C_tCUl2XZKRNtkWBy8I,21155
 frontend/styles.css,sha256=wly8O-zF4EUgV12Tv1bATSfmJsLITv2u3_SiyXVaxv4,4096
 frontend/utils.js,sha256=Ts4nlef8pkrEgpwX-uQwAhWvwxlIzex8ijDLNCa22ps,2372
 lfss/cli/balance.py,sha256=heOgwH6oNnfYsKJfA4VxWKdEXPstdVbbRXWxcDqLIS0,4176
-lfss/cli/cli.py,sha256=ehdsWJrIuIKEawAN9yVGUZjcVoIwwwZUqxLudQ7S9Ig,2259
+lfss/cli/cli.py,sha256=Yup3xIVEQPu10uM8dq1bvre1fK5ngweQHxXZsgQq4Hc,4187
 lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
 lfss/cli/serve.py,sha256=bO3GT0kuylMGN-7bZWP4e71MlugGZ_lEMkYaYld_Ntg,985
 lfss/cli/user.py,sha256=h-USWF6lB0Ztm9vwQznqsghKJ5INq5mBmaQeX2D5F-w,3490
-lfss/client/__init__.py,sha256=YttaGTBup7mxnW0CtxdZPF0HWga2cGM4twz9MXJIrts,1827
-lfss/client/api.py,sha256=x8-bb2K_dOvQPfNa7IsQE4H755iNlvkwzOPY7o9Bu-g,5638
+lfss/client/__init__.py,sha256=R9erioMInKIPZ0rXu1J-4mezbwyGMjig18532ATFP5s,4545
+lfss/client/api.py,sha256=aun5HWVNPBeJK6x0_iaM8gVcE3wX6yaqX0TsfsfifSw,5728
 lfss/sql/init.sql,sha256=C-JtQAlaOjESI8uoF1Y_9dKukEVSw5Ll-7yA3gG-XHU,1210
 lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lfss/src/config.py,sha256=Dn_94Wt8RHnJTSTawyh0UIhnpBmka7ajPkeRN1pCWTQ,651
+lfss/src/config.py,sha256=z0aVOW8yGgKSryhQCWTf2RY3iHmKMcKIZ-HiosTnPRs,539
 lfss/src/connection_pool.py,sha256=69QMJ4gRQ62qi39t0JKdvIaWRBrbU9S7slutIpCc30A,4959
-lfss/src/database.py,sha256=kDPkfA-h7n8c-8M0JMBhRGC7nH8s518W5NbNab5a7Wk,32147
+lfss/src/database.py,sha256=VKTLJSUFImF3pzMUaqGWEh1H06Yk3VgJa78bw5rAvG8,32147
 lfss/src/datatype.py,sha256=BLS7vuuKnFZQg0nrKeP9SymqUhcN6HwPgejU0yBd_Ak,1622
 lfss/src/error.py,sha256=imbhwnbhnI3HLhkbfICROe3F0gleKrOk4XnqHJDOtuI,285
 lfss/src/log.py,sha256=qNE04sHoZ2rMbQ5dR2zT7Xaz1KVAfYp5hKpWJX42S9g,5244
-lfss/src/server.py,sha256=J1pYq65thHC9ecFqeszHFoQuoFLEsgg6hWmW3XA9U-s,15447
+lfss/src/server.py,sha256=4GYDAa9Fx4H4fYh_He1u1zlIro3zaQmaEBNKpPO2Q2E,16374
 lfss/src/stat.py,sha256=hTMtQyM_Ukmhc33Bb9FGCfBMIX02KrGHQg8nL7sC8sU,2082
-lfss/src/utils.py,sha256=8VkrtpSmurbMiX7GyK-n7Grvzy3uwSJXHdONEsuLCDI,2272
-lfss-0.7.3.dist-info/METADATA,sha256=Ize9PqeFnq-Ki18BgTteSs9HSzaICY20mRUE35s9kV8,1967
-lfss-0.7.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.7.3.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
-lfss-0.7.3.dist-info/RECORD,,
+lfss/src/utils.py,sha256=ZE3isOS3gafEYw1z8s2ucY08eWQHIN4YUbKYH8F1hEQ,2409
+lfss-0.7.4.dist-info/METADATA,sha256=XeJ4ft1B9KDdfcxyFrFdwUMTYc1QsUAkgqI8v5rwbq4,1967
+lfss-0.7.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.7.4.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
+lfss-0.7.4.dist-info/RECORD,,