lfss 0.12.3__py3-none-any.whl → 0.13.0__py3-none-any.whl

lfss/api/__init__.py

@@ -1,203 +1,12 @@
-import os, time, pathlib
-from threading import Lock
-from .connector import Connector
-from ..eng.datatype import FileRecord
-from ..eng.utils import decode_uri_components
-from ..eng.bounded_pool import BoundedThreadPoolExecutor
 
-def upload_file(
-    connector: Connector, 
-    file_path: str, 
-    dst_url: str, 
-    n_retries: int = 0, 
-    interval: float = 0, 
-    verbose: bool = False, 
-    **put_kwargs
-    ) -> tuple[bool, str]:
-    this_try = 0
-    error_msg = ""
-    assert not file_path.endswith('/'), "File path must not end with a slash."
-    if dst_url.endswith('/'):
-        fname = file_path.split('/')[-1]
-        dst_url = f"{dst_url}{fname}"
+from .bundle import *
+from .connector import Client
 
-    while this_try <= n_retries:
-        try:
-            fsize = os.path.getsize(file_path)
-            if fsize < 32 * 1024 * 1024:     # 32MB
-                with open(file_path, 'rb') as f:
-                    blob = f.read()
-                connector.put(dst_url, blob, **put_kwargs)
-            else:
-                connector.post(dst_url, file_path, **put_kwargs)
-            break
-        except Exception as e:
-            if isinstance(e, KeyboardInterrupt):
-                raise e
-            if verbose:
-                print(f"Error uploading {file_path}: {e}, retrying...")
-            error_msg = str(e)
-            if hasattr(e, 'response'):
-                error_msg = f"{error_msg}, {e.response.text}"   # type: ignore
-            this_try += 1
-        finally:
-            time.sleep(interval)
+# Backward compatibility
+class Connector(Client): ...
 
-    if this_try > n_retries:
-        if verbose:
-            print(f"Failed to upload {file_path} after {n_retries} retries.")
-        return False, error_msg
-    return True, error_msg
-
-def upload_directory(
-    connector: Connector, 
-    directory: str, 
-    path: str, 
-    n_concurrent: int = 1,
-    n_retries: int = 0,
-    interval: float = 0,
-    verbose: bool = False,
-    **put_kwargs
-    ) -> list[tuple[str, str]]:
-    assert path.endswith('/'), "Path must end with a slash."
-    if path.startswith('/'):
-        path = path[1:]
-    directory = str(directory)
-    
-    _counter = 0
-    _counter_lock = Lock()
-
-    faild_items = []
-    def put_file(c: Connector, file_path):
-        with _counter_lock:
-            nonlocal _counter
-            _counter += 1
-            this_count = _counter
-            dst_path = f"{path}{os.path.relpath(file_path, directory)}"
-            if verbose:
-                print(f"[{this_count}] {file_path} -> {dst_path}")
-
-        if not (res:=upload_file(
-            c, file_path, dst_path, 
-            n_retries=n_retries, interval=interval, verbose=verbose, **put_kwargs
-            ))[0]:
-            faild_items.append((file_path, res[1]))
-
-    with connector.session(n_concurrent) as c:
-        with BoundedThreadPoolExecutor(n_concurrent) as executor:
-            for root, dirs, files in os.walk(directory):
-                for file in files:
-                    executor.submit(put_file, c, os.path.join(root, file))
-
-    return faild_items
-
-def download_file(
-    connector: Connector, 
-    src_url: str, 
-    file_path: str, 
-    n_retries: int = 0, 
-    interval: float = 0, 
-    verbose: bool = False, 
-    overwrite: bool = False
-    ) -> tuple[bool, str]:
-    this_try = 0
-    error_msg = ""
-    assert not src_url.endswith('/'), "Source URL must not end with a slash."
-    while this_try <= n_retries:
-        if os.path.isdir(file_path):
-            fname = decode_uri_components(src_url.split('/')[-1])
-            file_path = os.path.join(file_path, fname)
-
-        if not overwrite and os.path.exists(file_path):
-            if verbose:
-                print(f"File {file_path} already exists, skipping download.")
-            return True, error_msg
-        try:
-            fmeta = connector.get_meta(src_url)
-            if fmeta is None:
-                error_msg = "File not found."
-                return False, error_msg
-
-            pathlib.Path(file_path).parent.mkdir(parents=True, exist_ok=True)
-            fsize = fmeta.file_size   # type: ignore
-            if fsize < 32 * 1024 * 1024:     # 32MB
-                blob = connector.get(src_url)
-                assert blob is not None
-                with open(file_path, 'wb') as f:
-                    f.write(blob)
-            else:
-                with open(file_path, 'wb') as f:
-                    for chunk in connector.get_stream(src_url):
-                        f.write(chunk)
-            break
-
-        except Exception as e:
-            if isinstance(e, KeyboardInterrupt):
-                raise e
-            if verbose:
-                print(f"Error downloading {src_url}: {e}, retrying...")
-            error_msg = str(e)
-            if hasattr(e, 'response'):
-                error_msg = f"{error_msg}, {e.response.text}"   # type: ignore
-            this_try += 1
-        finally:
-            time.sleep(interval)
-
-    if this_try > n_retries:
-        if verbose:
-            print(f"Failed to download {src_url} after {n_retries} retries.")
-        return False, error_msg
-    return True, error_msg
-
-def download_directory(
-    connector: Connector,
-    src_path: str,
-    directory: str,
-    n_concurrent: int = 1,
-    n_retries: int = 0,
-    interval: float = 0,
-    verbose: bool = False,
-    overwrite: bool = False
-    ) -> list[tuple[str, str]]:
-
-    directory = str(directory)
-
-    if not src_path.endswith('/'):
-        src_path += '/'
-    if not directory.endswith(os.sep):
-        directory += os.sep
-    
-    _counter = 0
-    _counter_lock = Lock()
-    failed_items: list[tuple[str, str]] = []
-    file_count = 0
-    def get_file(c, src_url):
-        nonlocal _counter, failed_items, file_count, verbose
-        with _counter_lock:
-            _counter += 1
-            this_count = _counter
-            dst_path = f"{directory}{os.path.relpath(decode_uri_components(src_url), decode_uri_components(src_path))}"
-            if verbose:
-                print(f"[{this_count}/{file_count}] {src_url} -> {dst_path}")
-
-        if not (res:=download_file(
-            c, src_url, dst_path, 
-            n_retries=n_retries, interval=interval, verbose=verbose, overwrite=overwrite
-            ))[0]:
-            failed_items.append((src_url, res[1]))
-        
-    batch_size = 10_000
-    file_list: list[FileRecord] = []
-    with connector.session(n_concurrent) as c:
-        file_count = c.count_files(src_path, flat=True)
-        for offset in range(0, file_count, batch_size):
-            if verbose:
-                print(f"Retrieving file list... ({offset}/{file_count})", end='\r')
-            file_list.extend(c.list_files(
-                src_path, offset=offset, limit=batch_size, flat=True
-            ))
-
-        with BoundedThreadPoolExecutor(n_concurrent) as executor:
-            for file in file_list:
-                executor.submit(get_file, c, file.url)
-    return failed_items
+__all__ = [
+    "upload_file", "upload_directory",
+    "download_file", "download_directory", 
+    "Client", "Connector",
+]

lfss/api/bundle.py

@@ -0,0 +1,201 @@
+import os, time, pathlib
+from threading import Lock
+from .connector import Client
+from ..eng.datatype import FileRecord
+from ..eng.utils import decode_uri_components
+from ..eng.bounded_pool import BoundedThreadPoolExecutor
+
+def upload_file(
+    connector: Client, 
+    file_path: str, 
+    dst_url: str, 
+    n_retries: int = 0, 
+    interval: float = 0, 
+    verbose: bool = False, 
+    **put_kwargs
+    ) -> tuple[bool, str]:
+    this_try = 0
+    error_msg = ""
+    assert not file_path.endswith('/'), "File path must not end with a slash."
+    if dst_url.endswith('/'):
+        fname = file_path.split('/')[-1]
+        dst_url = f"{dst_url}{fname}"
+
+    while this_try <= n_retries:
+        try:
+            fsize = os.path.getsize(file_path)
+            if fsize < 32 * 1024 * 1024:     # 32MB
+                with open(file_path, 'rb') as f:
+                    blob = f.read()
+                connector.put(dst_url, blob, **put_kwargs)
+            else:
+                connector.post(dst_url, file_path, **put_kwargs)
+            break
+        except Exception as e:
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+            if verbose:
+                print(f"Error uploading {file_path}: {e}, retrying...")
+            error_msg = str(e)
+            if hasattr(e, 'response'):
+                error_msg = f"{error_msg}, {e.response.text}"   # type: ignore
+            this_try += 1
+        finally:
+            time.sleep(interval)
+
+    if this_try > n_retries:
+        if verbose:
+            print(f"Failed to upload {file_path} after {n_retries} retries.")
+        return False, error_msg
+    return True, error_msg
+
+def upload_directory(
+    connector: Client, 
+    directory: str, 
+    path: str, 
+    n_concurrent: int = 1,
+    n_retries: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    **put_kwargs
+    ) -> list[tuple[str, str]]:
+    assert path.endswith('/'), "Path must end with a slash."
+    if path.startswith('/'):
+        path = path[1:]
+    directory = str(directory)
+    
+    _counter = 0
+    _counter_lock = Lock()
+
+    faild_items = []
+    def put_file(c: Client, file_path):
+        with _counter_lock:
+            nonlocal _counter
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{path}{os.path.relpath(file_path, directory)}"
+            if verbose:
+                print(f"[{this_count}] {file_path} -> {dst_path}")
+
+        if not (res:=upload_file(
+            c, file_path, dst_path, 
+            n_retries=n_retries, interval=interval, verbose=verbose, **put_kwargs
+            ))[0]:
+            faild_items.append((file_path, res[1]))
+
+    with connector.session(n_concurrent) as c, BoundedThreadPoolExecutor(n_concurrent) as executor:
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                executor.submit(put_file, c, os.path.join(root, file))
+
+    return faild_items
+
+def download_file(
+    connector: Client, 
+    src_url: str, 
+    file_path: str, 
+    n_retries: int = 0, 
+    interval: float = 0, 
+    verbose: bool = False, 
+    overwrite: bool = False
+    ) -> tuple[bool, str]:
+    this_try = 0
+    error_msg = ""
+    assert not src_url.endswith('/'), "Source URL must not end with a slash."
+    while this_try <= n_retries:
+        if os.path.isdir(file_path):
+            fname = decode_uri_components(src_url.split('/')[-1])
+            file_path = os.path.join(file_path, fname)
+
+        if not overwrite and os.path.exists(file_path):
+            if verbose:
+                print(f"File {file_path} already exists, skipping download.")
+            return True, error_msg
+        try:
+            fmeta = connector.get_meta(src_url)
+            if fmeta is None:
+                error_msg = "File not found."
+                return False, error_msg
+
+            pathlib.Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+            fsize = fmeta.file_size   # type: ignore
+            if fsize < 32 * 1024 * 1024:     # 32MB
+                blob = connector.get(src_url)
+                assert blob is not None
+                with open(file_path, 'wb') as f:
+                    f.write(blob)
+            else:
+                with open(file_path, 'wb') as f:
+                    for chunk in connector.get_stream(src_url):
+                        f.write(chunk)
+            break
+
+        except Exception as e:
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+            if verbose:
+                print(f"Error downloading {src_url}: {e}, retrying...")
+            error_msg = str(e)
+            if hasattr(e, 'response'):
+                error_msg = f"{error_msg}, {e.response.text}"   # type: ignore
+            this_try += 1
+        finally:
+            time.sleep(interval)
+
+    if this_try > n_retries:
+        if verbose:
+            print(f"Failed to download {src_url} after {n_retries} retries.")
+        return False, error_msg
+    return True, error_msg
+
+def download_directory(
+    connector: Client,
+    src_path: str,
+    directory: str,
+    n_concurrent: int = 1,
+    n_retries: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    overwrite: bool = False
+    ) -> list[tuple[str, str]]:
+
+    directory = str(directory)
+
+    if not src_path.endswith('/'):
+        src_path += '/'
+    if not directory.endswith(os.sep):
+        directory += os.sep
+    
+    _counter = 0
+    _counter_lock = Lock()
+    failed_items: list[tuple[str, str]] = []
+    file_count = 0
+    def get_file(c, src_url):
+        nonlocal _counter, failed_items, file_count, verbose
+        with _counter_lock:
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{directory}{os.path.relpath(decode_uri_components(src_url), decode_uri_components(src_path))}"
+            if verbose:
+                print(f"[{this_count}/{file_count}] {src_url} -> {dst_path}")
+
+        if not (res:=download_file(
+            c, src_url, dst_path, 
+            n_retries=n_retries, interval=interval, verbose=verbose, overwrite=overwrite
+            ))[0]:
+            failed_items.append((src_url, res[1]))
+        
+    batch_size = 10_000
+    file_list: list[FileRecord] = []
+    with connector.session(n_concurrent) as c, BoundedThreadPoolExecutor(n_concurrent) as executor:
+        file_count = c.count_files(src_path, flat=True)
+        for offset in range(0, file_count, batch_size):
+            if verbose:
+                print(f"Retrieving file list... ({offset}/{file_count})", end='\r')
+            file_list.extend(c.list_files(
+                src_path, offset=offset, limit=batch_size, flat=True
+            ))
+
+        for file in file_list:
+            executor.submit(get_file, c, file.url)
+    return failed_items

lfss/api/connector.py

@@ -7,7 +7,6 @@ import requests.adapters
 import urllib.parse
 from tempfile import SpooledTemporaryFile
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from lfss.eng.error import PathNotFoundError
 from lfss.eng.datatype import (
     FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents, AccessLevel, 
     FileSortKey, DirSortKey
@@ -25,34 +24,34 @@ def _p(x: str) -> str:
         x = x[1:]
     return x
 
-class Connector:
+class Client:
     class Session:
         def __init__(
-            self, connector: Connector, pool_size: int = 10, 
+            self, client: Client, pool_size: int = 10, 
             retry: int = 1, backoff_factor: num_t = 0.5, status_forcelist: list[int] = [503]
             ):
-            self.connector = connector
+            self.client = client
             self.pool_size = pool_size
             self.retry_adapter = requests.adapters.Retry(
                 total=retry, backoff_factor=backoff_factor, status_forcelist=status_forcelist, 
             )
         def open(self):
             self.close()
-            if self.connector._session is None:
+            if self.client._session is None:
                 s = requests.Session()
                 adapter = requests.adapters.HTTPAdapter(pool_connections=self.pool_size, pool_maxsize=self.pool_size, max_retries=self.retry_adapter)
                 s.mount('http://', adapter)
                 s.mount('https://', adapter)
-                self.connector._session = s
+                self.client._session = s
         def close(self):
-            if self.connector._session is not None:
-                self.connector._session.close()
-            self.connector._session = None
+            if self.client._session is not None:
+                self.client._session.close()
+            self.client._session = None
         def __call__(self):
-            return self.connector
+            return self.client
         def __enter__(self):
             self.open()
-            return self.connector
+            return self.client
         def __exit__(self, exc_type, exc_value, traceback):
             self.close()
 
@@ -77,7 +76,7 @@ class Connector:
         return self.Session(self, pool_size, **kwargs)
     
     def _fetch_factory(
-        self, method: Literal['GET', 'POST', 'PUT', 'DELETE'], 
+        self, method: Literal['GET', 'POST', 'PUT', 'DELETE', 'HEAD'], 
         path: str, search_params: dict = {}, extra_headers: dict = {}
     ):
         if path.startswith('/'):
@@ -103,6 +102,17 @@ class Connector:
                     response.raise_for_status()
             return response
         return f
+    
+    def exists(self, path: str) -> bool:
+        """Checks if a file/directory exists."""
+        path = _p(path)
+        try:
+            self._fetch_factory('HEAD', path)()
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                return False
+            raise e
+        return True
 
     def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip', 'skip-ahead'] = 'abort'):
         """Uploads a file to the specified path."""
@@ -184,23 +194,16 @@ class Connector:
         )
         return response.json()
     
-    def _get(self, path: str, stream: bool = False) -> Optional[requests.Response]:
-        try:
-            response = self._fetch_factory('GET', path)(stream=stream)
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 404:
-                return None
-            raise e
-        return response
+    def _get(self, path: str, stream: bool = False) -> requests.Response:
+        return self._fetch_factory('GET', path)(stream=stream)
 
-    def get(self, path: str) -> Optional[bytes]:
+    def get(self, path: str) -> bytes:
         """Downloads a file from the specified path."""
         path = _p(path)
         response = self._get(path)
-        if response is None: return None
         return response.content
 
-    def get_partial(self, path: str, range_start: int = -1, range_end: int = -1) -> Optional[bytes]:
+    def get_partial(self, path: str, range_start: int = -1, range_end: int = -1) -> bytes:
         """
         Downloads a partial file from the specified path.
         start and end are the byte offsets, both inclusive.
@@ -209,20 +212,16 @@ class Connector:
         response = self._fetch_factory('GET', path, extra_headers={
             'Range': f"bytes={range_start if range_start >= 0 else ''}-{range_end if range_end >= 0 else ''}"
         })()
-        if response is None: return None
         return response.content
     
     def get_stream(self, path: str, chunk_size = 1024) -> Iterator[bytes]:
         """Downloads a file from the specified path, will raise PathNotFoundError if path not found."""
         path = _p(path)
-        response = self._get(path, stream=True)
-        if response is None: raise PathNotFoundError("Path not found: " + path)
-        return response.iter_content(chunk_size)
+        return self._get(path, stream=True).iter_content(chunk_size)
 
-    def get_json(self, path: str) -> Optional[dict]:
+    def get_json(self, path: str) -> dict:
         path = _p(path)
         response = self._get(path)
-        if response is None: return None
         assert response.headers['Content-Type'] == 'application/json'
         return response.json()
     
@@ -242,23 +241,18 @@ class Connector:
         path = _p(path)
         self._fetch_factory('DELETE', path)()
     
-    def get_meta(self, path: str) -> Optional[FileRecord | DirectoryRecord]:
+    def get_meta(self, path: str) -> FileRecord | DirectoryRecord:
         """Gets the metadata for the file at the specified path."""
         path = _p(path)
-        try:
-            response = self._fetch_factory('GET', '_api/meta', {'path': path})()
-            if path.endswith('/'):
-                return DirectoryRecord(**response.json())
-            else:
-                return FileRecord(**response.json())
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 404:
-                return None
-            raise e
+        response = self._fetch_factory('GET', '_api/meta', {'path': path})()
+        if path.endswith('/'):
+            return DirectoryRecord(**response.json())
+        else:
+            return FileRecord(**response.json())
     # shorthand methods for type constraints
-    def get_fmeta(self, path: str) -> Optional[FileRecord]: assert (f:=self.get_meta(path)) is None or isinstance(f, FileRecord); return f 
-    def get_dmeta(self, path: str) -> Optional[DirectoryRecord]: assert (d:=self.get_meta(path)) is None or isinstance(d, DirectoryRecord); return d
-    
+    def get_fmeta(self, path: str) -> FileRecord: assert (f:=self.get_meta(path)) is None or isinstance(f, FileRecord); return f
+    def get_dmeta(self, path: str) -> DirectoryRecord: assert (d:=self.get_meta(path)) is None or isinstance(d, DirectoryRecord); return d
+
     def count_files(self, path: str, flat: bool = False) -> int:
         assert path.endswith('/')
         path = _p(path)
@@ -307,11 +301,11 @@ class Connector:
         path = _p(path)
         if path == '/':
             # handle root path separately
-            # TODO: change later
-            response = self._fetch_factory('GET', path)()
-            dirs = [DirectoryRecord(**d) for d in response.json()['dirs']]
-            files = [FileRecord(**f) for f in response.json()['files']]
-            return PathContents(dirs=dirs, files=files)
+            dirnames = [f'{self.whoami().username}/'] + [f'{p.username}/' for p in self.list_peers(AccessLevel.READ)]
+            return PathContents(
+                dirs = [DirectoryRecord(url = d) for d in dirnames], 
+                files = []
+            )
 
         dirs: list[DirectoryRecord] = []
         files: list[FileRecord] = []
@@ -360,14 +354,14 @@ class Connector:
     def set_file_permission(self, path: str, permission: int | FileReadPermission):
         """Sets the file permission for the specified path."""
         path = _p(path)
-        self._fetch_factory('POST', '_api/meta', {'path': path, 'perm': int(permission)})(
+        self._fetch_factory('POST', '_api/set-perm', {'path': path, 'perm': int(permission)})(
             headers={'Content-Type': 'application/www-form-urlencoded'}
         )
         
     def move(self, path: str, new_path: str):
         """Move file or directory to a new path."""
         path = _p(path); new_path = _p(new_path)
-        self._fetch_factory('POST', '_api/meta', {'path': path, 'new_path': new_path})(
+        self._fetch_factory('POST', '_api/move', {'src': path, 'dst': new_path})(
             headers = {'Content-Type': 'application/www-form-urlencoded'}
         )
     
@@ -389,11 +383,21 @@ class Connector:
         
     def whoami(self) -> UserRecord:
         """Gets information about the current user."""
-        response = self._fetch_factory('GET', '_api/whoami')()
+        response = self._fetch_factory('GET', '_api/user/whoami')()
         return UserRecord(**response.json())
+    
+    def storage_used(self) -> int:
+        """Gets the storage used by the current user, in bytes."""
+        response = self._fetch_factory('GET', '_api/user/storage')()
+        return response.json()['used']
 
     def list_peers(self, level: AccessLevel = AccessLevel.READ, incoming: bool = False) -> list[UserRecord]:
-        """List all users that have at least the given access level to the current user."""
+        """
+        if incoming is False (default): 
+            list all users that the current user has at least the given access level to, 
+        if incoming is True: 
+            list all users that have at least the given access level to the current user
+        """
         response = self._fetch_factory('GET', '_api/list-peers', {'level': int(level), 'incoming': incoming})()
         users = [UserRecord(**u) for u in response.json()]
         return users

lfss/cli/__init__.py

@@ -3,17 +3,24 @@ from typing import Iterable, TypeVar, Generator, Callable, Optional
 import requests, os
 
 @contextmanager
-def catch_request_error(error_code_handler: Optional[ dict[int, Callable[[requests.Response], None]] ] = None):
+def catch_request_error(
+    error_code_handler: Optional[ dict[int, Callable[[requests.Response], None]] ] = None, 
+    cleanup_fn: Optional[Callable[[], None]] = None
+    ):
     try:
         yield
     except requests.RequestException as e:
         if error_code_handler is not None:
             if e.response is not None and e.response.status_code in error_code_handler:
                 error_code_handler[e.response.status_code](e.response)
+                if cleanup_fn is not None:
+                    cleanup_fn()
                 return
         print(f"\033[31m[Request error]: {e}\033[0m")
         if e.response is not None:
             print(f"\033[91m[Error message]: {e.response.text}\033[0m")
+        if cleanup_fn is not None:
+            cleanup_fn()
 
 T = TypeVar('T')
 def line_sep(iter: Iterable[T], enable=True, start=True, end=True, middle=False, color="\033[90m") -> Generator[T, None, None]: