pybiolib 1.1.1747__py3-none-any.whl → 1.1.2193__py3-none-any.whl

biolib/_internal/fuse_mount/experiment_fuse_mount.py

@@ -0,0 +1,209 @@
+import errno
+import os
+import stat
+from datetime import datetime, timezone
+from time import time
+
+from biolib._internal.libs.fusepy import FUSE, FuseOSError, Operations
+from biolib.biolib_errors import BioLibError
+from biolib.jobs import Job
+from biolib.typing_utils import Dict, List, Optional, Tuple, TypedDict
+
+
+class _AttributeDict(TypedDict):
+    st_atime: int
+    st_ctime: int
+    st_gid: int
+    st_mode: int
+    st_mtime: int
+    st_nlink: int
+    st_size: int
+    st_uid: int
+
+
+_SUCCESS_CODE = 0
+
+
+class ExperimentFuseMount(Operations):
+    def __init__(self, experiment):
+        self._experiment = experiment
+        self._job_names_map: Optional[Dict[str, Job]] = None
+        self._jobs_last_fetched_at: float = 0.0
+        self._mounted_at_epoch_seconds: int = int(time())
+
+    @staticmethod
+    def mount_experiment(experiment, mount_path: str) -> None:
+        FUSE(
+            operations=ExperimentFuseMount(experiment),
+            mountpoint=mount_path,
+            nothreads=True,
+            foreground=True,
+            allow_other=False,
+        )
+
+    def getattr(self, path: str, fh=None) -> _AttributeDict:
+        if path == '/':
+            return self._get_directory_attributes(timestamp_epoch_seconds=self._mounted_at_epoch_seconds)
+
+        job, path_in_job = self._parse_path(path)
+        job_finished_at_epoch_seconds: int = int(
+            datetime.fromisoformat(job.to_dict()['finished_at'].rstrip('Z')).replace(tzinfo=timezone.utc).timestamp()
+        )
+
+        if path_in_job == '/':
+            return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
+
+        try:
+            file = job.get_output_file(path_in_job)
+            return self._get_file_attributes(
+                timestamp_epoch_seconds=job_finished_at_epoch_seconds,
+                size_in_bytes=file.length,
+            )
+        except BioLibError:
+            # file not found
+            pass
+
+        file_paths_in_job = [file.path for file in job.list_output_files()]
+
+        for file_path_in_job in file_paths_in_job:
+            if file_path_in_job.startswith(path_in_job):
+                return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
+
+        raise FuseOSError(errno.ENOENT) from None  # No such file or directory
+
+    def readdir(self, path: str, fh: int) -> List[str]:
+        directory_entries = ['.', '..']
+
+        if path == '/':
+            directory_entries.extend(self._get_job_names_map(refresh_jobs=True).keys())
+        else:
+            job, path_in_job = self._parse_path(path)
+            dir_path_in_job = '/' if path_in_job == '/' else path_in_job + '/'
+            depth = dir_path_in_job.count('/')
+            directory_entries.extend(
+                set(
+                    [
+                        file.path.split('/')[depth]
+                        for file in job.list_output_files()
+                        if file.path.startswith(dir_path_in_job)
+                    ]
+                )
+            )
+
+        return directory_entries
+
+    def open(self, path: str, flags: int) -> int:
+        job, path_in_job = self._parse_path(path)
+        try:
+            job.get_output_file(path_in_job)
+        except BioLibError:
+            # file not found
+            raise FuseOSError(errno.ENOENT) from None
+
+        return 1234  # dummy file handle
+
+    def read(self, path: str, size: int, offset: int, fh: int) -> bytes:
+        job, path_in_job = self._parse_path(path)
+        try:
+            file = job.get_output_file(path_in_job)
+        except BioLibError:
+            raise FuseOSError(errno.ENOENT) from None  # No such file or directory
+
+        return file.get_data(start=offset, length=size)
+
+    def release(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    def releasedir(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    def flush(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    @staticmethod
+    def _get_directory_attributes(timestamp_epoch_seconds: int) -> _AttributeDict:
+        return _AttributeDict(
+            st_atime=timestamp_epoch_seconds,
+            st_ctime=timestamp_epoch_seconds,
+            st_gid=os.getgid(),
+            st_mode=stat.S_IFDIR | 0o555,  # Directory that is readable and executable by owner, group, and others.
+            st_mtime=timestamp_epoch_seconds,
+            st_nlink=1,
+            st_size=1,
+            st_uid=os.getuid(),
+        )
+
+    @staticmethod
+    def _get_file_attributes(timestamp_epoch_seconds: int, size_in_bytes: int) -> _AttributeDict:
+        return _AttributeDict(
+            st_atime=timestamp_epoch_seconds,
+            st_ctime=timestamp_epoch_seconds,
+            st_gid=os.getgid(),
+            st_mode=stat.S_IFREG | 0o444,  # Regular file with read permissions for owner, group, and others.
+            st_mtime=timestamp_epoch_seconds,
+            st_nlink=1,
+            st_size=size_in_bytes,
+            st_uid=os.getuid(),
+        )
+
+    def _get_job_names_map(self, refresh_jobs=False) -> Dict[str, Job]:
+        current_time = time()
+        if not self._job_names_map or (current_time - self._jobs_last_fetched_at > 1 and refresh_jobs):
+            self._jobs_last_fetched_at = current_time
+            self._job_names_map = {job.get_name(): job for job in self._experiment.get_jobs(status='completed')}
+
+        return self._job_names_map
+
+    def _parse_path(self, path: str) -> Tuple[Job, str]:
+        path_splitted = path.split('/')
+        job_name = path_splitted[1]
+        path_in_job = '/' + '/'.join(path_splitted[2:])
+        job = self._get_job_names_map().get(job_name)
+        if not job:
+            raise FuseOSError(errno.ENOENT)  # No such file or directory
+
+        return job, path_in_job
+
+    # ----------------------------------- File system methods not implemented below -----------------------------------
+
+    def chmod(self, path, mode):
+        raise FuseOSError(errno.EACCES)
+
+    def chown(self, path, uid, gid):
+        raise FuseOSError(errno.EACCES)
+
+    def mknod(self, path, mode, dev):
+        raise FuseOSError(errno.EACCES)
+
+    def rmdir(self, path):
+        raise FuseOSError(errno.EACCES)
+
+    def mkdir(self, path, mode):
+        raise FuseOSError(errno.EACCES)
+
+    def unlink(self, path):
+        raise FuseOSError(errno.EACCES)
+
+    def symlink(self, target, source):
+        raise FuseOSError(errno.EACCES)
+
+    def rename(self, old, new):
+        raise FuseOSError(errno.EACCES)
+
+    def link(self, target, source):
+        raise FuseOSError(errno.EACCES)
+
+    def utimens(self, path, times=None):
+        raise FuseOSError(errno.EACCES)
+
+    def create(self, path, mode, fi=None):
+        raise FuseOSError(errno.EACCES)
+
+    def write(self, path, data, offset, fh):
+        raise FuseOSError(errno.EACCES)
+
+    def truncate(self, path, length, fh=None):
+        raise FuseOSError(errno.EACCES)
+
+    def fsync(self, path, datasync, fh):
+        raise FuseOSError(errno.EACCES)

biolib/_internal/http_client.py

@@ -1,25 +1,25 @@
 import json
 import platform
-import time
+import shutil
 import socket
 import ssl
 import subprocess
-import urllib.request
+import time
 import urllib.error
 import urllib.parse
+import urllib.request
 
 from biolib.biolib_logging import logger_no_user_data
-from biolib.typing_utils import Dict, Optional, Union, Literal, cast
+from biolib.typing_utils import Dict, Literal, Optional, Union, cast
+
+_HttpMethod = Literal['GET', 'POST', 'PATCH', 'PUT']
 
 
 def _create_ssl_context():
     context = ssl.create_default_context()
     try:
         if platform.system() == 'Darwin':
-            certificates = subprocess.check_output(
-                "security find-certificate -a -p",
-                shell=True
-            ).decode('utf-8')
+            certificates = subprocess.check_output('security find-certificate -a -p', shell=True).decode('utf-8')
             context.load_verify_locations(cadata=certificates)
     except BaseException:
         pass
@@ -33,7 +33,7 @@ class HttpError(urllib.error.HTTPError):
             code=http_error.code,
             msg=http_error.msg,  # type: ignore
             hdrs=http_error.hdrs,  # type: ignore
-            fp=http_error.fp
+            fp=http_error.fp,
         )
 
     def __str__(self):
@@ -42,15 +42,24 @@ class HttpError(urllib.error.HTTPError):
 
 
 class HttpResponse:
-    def __init__(self, response):
+    def __init__(self, response, response_path) -> None:
         self.headers: Dict[str, str] = dict(response.headers)
         self.status_code: int = int(response.status)
-        self.content: bytes = response.read()
+        self.response_path = response_path
+        if self.response_path:
+            with open(self.response_path, 'wb') as out_file:
+                shutil.copyfileobj(response, out_file)
+        else:
+            self.content: bytes = response.read()
         self.url: str = response.geturl()
 
     @property
     def text(self) -> str:
-        return cast(str, self.content.decode('utf-8'))
+        if self.response_path:
+            with open(self.response_path, 'rb') as fp:
+                return cast(str, fp.read().decode('utf-8'))
+        else:
+            return cast(str, self.content.decode('utf-8'))
 
     def json(self):
         return json.loads(self.text)
@@ -61,12 +70,13 @@ class HttpClient:
 
     @staticmethod
     def request(
-            url: str,
-            method: Optional[Literal['GET', 'POST', 'PATCH', 'PUT']] = None,
-            data: Optional[Union[Dict, bytes]] = None,
-            headers: Optional[Dict[str, str]] = None,
-            retries: int = 5,
-            timeout_in_seconds: Optional[int] = None,
+        url: str,
+        method: Optional[_HttpMethod] = None,
+        data: Optional[Union[Dict, bytes]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        retries: int = 5,
+        timeout_in_seconds: Optional[int] = None,
+        response_path: Optional[str] = None,
     ) -> HttpResponse:
         if not HttpClient.ssl_context:
             HttpClient.ssl_context = _create_ssl_context()
@@ -84,7 +94,7 @@ class HttpClient:
         if timeout_in_seconds is None:
             timeout_in_seconds = 60 if isinstance(data, dict) else 180  # TODO: Calculate timeout based on data size
 
-        last_error: Optional[urllib.error.URLError] = None
+        last_error: Optional[Exception] = None
         for retry_count in range(retries + 1):
             if retry_count > 0:
                 time.sleep(5 * retry_count)
@@ -95,23 +105,32 @@ class HttpClient:
                     context=HttpClient.ssl_context,
                     timeout=timeout_in_seconds,
                 ) as response:
-                    return HttpResponse(response)
+                    return HttpResponse(response, response_path)
 
             except urllib.error.HTTPError as error:
-                if error.code == 502:
-                    logger_no_user_data.debug(f'HTTP {method} request failed with status 502 for "{url}"')
+                if error.code == 429:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 429 for "{url}"')
+                    last_error = error
+                elif error.code == 502:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 502 for "{url}"')
                     last_error = error
                 elif error.code == 503:
-                    logger_no_user_data.debug(f'HTTP {method} request failed with status 503 for "{url}"')
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 503 for "{url}"')
+                    last_error = error
+                elif error.code == 504:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 504 for "{url}"')
                     last_error = error
                 else:
                     raise HttpError(error) from None
 
             except urllib.error.URLError as error:
                 if isinstance(error.reason, socket.timeout):
-                    logger_no_user_data.debug(f'HTTP {method} request failed with read timeout for "{url}"')
+                    logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
                     last_error = error
                 else:
                     raise error
+            except socket.timeout as error:
+                logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
+                last_error = error
 
         raise last_error or Exception(f'HTTP {method} request failed after {retries} retries for "{url}"')

biolib/_internal/lfs/__init__.py

@@ -0,0 +1 @@
+from .cache import prune_lfs_cache

biolib/_internal/libs/__init__.py

@@ -0,0 +1 @@
+# Note: this directory is purely for libraries to be directly included instead of as dependencies