pybiolib 1.1.1881__py3-none-any.whl → 1.2.7.dev0__py3-none-any.whl

biolib/_internal/file_utils.py

@@ -0,0 +1,77 @@
+import io
+import os
+import zipfile as zf
+from pathlib import Path
+
+from biolib.typing_utils import Iterator, List, Tuple
+
+
+def get_files_and_size_of_directory(directory: str) -> Tuple[List[str], int]:
+    data_size = 0
+    file_list: List[str] = []
+
+    for path, _, files in os.walk(directory):
+        for file in files:
+            file_path = os.path.join(path, file)
+            if os.path.islink(file_path):
+                continue  # skip symlinks
+
+            relative_file_path = file_path[len(directory) + 1 :]  # +1 to remove starting slash
+            file_list.append(relative_file_path)
+            data_size += os.path.getsize(file_path)
+
+    return file_list, data_size
+
+
+def get_iterable_zip_stream(files: List[str], chunk_size: int) -> Iterator[bytes]:
+    class ChunkedIOBuffer(io.RawIOBase):
+        def __init__(self, chunk_size: int):
+            super().__init__()
+            self.chunk_size = chunk_size
+            self.tmp_data = bytearray()
+
+        def get_buffer_size(self):
+            return len(self.tmp_data)
+
+        def read_chunk(self):
+            chunk = bytes(self.tmp_data[: self.chunk_size])
+            self.tmp_data = self.tmp_data[self.chunk_size :]
+            return chunk
+
+        def write(self, data):
+            data_length = len(data)
+            self.tmp_data += data
+            return data_length
+
+    # create chunked buffer to hold data temporarily
+    io_buffer = ChunkedIOBuffer(chunk_size)
+
+    # create zip writer that will write to the io buffer
+    zip_writer = zf.ZipFile(io_buffer, mode='w')  # type: ignore
+
+    for file_path in files:
+        # generate zip info and prepare zip pointer for writing
+        z_info = zf.ZipInfo.from_file(file_path)
+        zip_pointer = zip_writer.open(z_info, mode='w')
+        if Path(file_path).is_file():
+            # read file chunk by chunk
+            with open(file_path, 'br') as file_pointer:
+                while True:
+                    chunk = file_pointer.read(chunk_size)
+                    if len(chunk) == 0:
+                        break
+                    # write the chunk to the zip
+                    zip_pointer.write(chunk)
+                    # if writing the chunk caused us to go over chunk_size, flush it
+                    if io_buffer.get_buffer_size() > chunk_size:
+                        yield io_buffer.read_chunk()
+
+        zip_pointer.close()
+
+    # flush any remaining data in the stream (e.g. zip file meta data)
+    zip_writer.close()
+    while True:
+        chunk = io_buffer.read_chunk()
+        if len(chunk) == 0:
+            break
+        yield chunk

biolib/_internal/fuse_mount/__init__.py

@@ -0,0 +1 @@
+from .experiment_fuse_mount import ExperimentFuseMount

biolib/_internal/fuse_mount/experiment_fuse_mount.py

@@ -0,0 +1,209 @@
+import errno
+import os
+import stat
+from datetime import datetime, timezone
+from time import time
+
+from biolib._internal.libs.fusepy import FUSE, FuseOSError, Operations
+from biolib.biolib_errors import BioLibError
+from biolib.jobs import Job
+from biolib.typing_utils import Dict, List, Optional, Tuple, TypedDict
+
+
+class _AttributeDict(TypedDict):
+    st_atime: int
+    st_ctime: int
+    st_gid: int
+    st_mode: int
+    st_mtime: int
+    st_nlink: int
+    st_size: int
+    st_uid: int
+
+
+_SUCCESS_CODE = 0
+
+
+class ExperimentFuseMount(Operations):
+    def __init__(self, experiment):
+        self._experiment = experiment
+        self._job_names_map: Optional[Dict[str, Job]] = None
+        self._jobs_last_fetched_at: float = 0.0
+        self._mounted_at_epoch_seconds: int = int(time())
+
+    @staticmethod
+    def mount_experiment(experiment, mount_path: str) -> None:
+        FUSE(
+            operations=ExperimentFuseMount(experiment),
+            mountpoint=mount_path,
+            nothreads=True,
+            foreground=True,
+            allow_other=False,
+        )
+
+    def getattr(self, path: str, fh=None) -> _AttributeDict:
+        if path == '/':
+            return self._get_directory_attributes(timestamp_epoch_seconds=self._mounted_at_epoch_seconds)
+
+        job, path_in_job = self._parse_path(path)
+        job_finished_at_epoch_seconds: int = int(
+            datetime.fromisoformat(job.to_dict()['finished_at'].rstrip('Z')).replace(tzinfo=timezone.utc).timestamp()
+        )
+
+        if path_in_job == '/':
+            return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
+
+        try:
+            file = job.get_output_file(path_in_job)
+            return self._get_file_attributes(
+                timestamp_epoch_seconds=job_finished_at_epoch_seconds,
+                size_in_bytes=file.length,
+            )
+        except BioLibError:
+            # file not found
+            pass
+
+        file_paths_in_job = [file.path for file in job.list_output_files()]
+
+        for file_path_in_job in file_paths_in_job:
+            if file_path_in_job.startswith(path_in_job):
+                return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
+
+        raise FuseOSError(errno.ENOENT) from None  # No such file or directory
+
+    def readdir(self, path: str, fh: int) -> List[str]:
+        directory_entries = ['.', '..']
+
+        if path == '/':
+            directory_entries.extend(self._get_job_names_map(refresh_jobs=True).keys())
+        else:
+            job, path_in_job = self._parse_path(path)
+            dir_path_in_job = '/' if path_in_job == '/' else path_in_job + '/'
+            depth = dir_path_in_job.count('/')
+            directory_entries.extend(
+                set(
+                    [
+                        file.path.split('/')[depth]
+                        for file in job.list_output_files()
+                        if file.path.startswith(dir_path_in_job)
+                    ]
+                )
+            )
+
+        return directory_entries
+
+    def open(self, path: str, flags: int) -> int:
+        job, path_in_job = self._parse_path(path)
+        try:
+            job.get_output_file(path_in_job)
+        except BioLibError:
+            # file not found
+            raise FuseOSError(errno.ENOENT) from None
+
+        return 1234  # dummy file handle
+
+    def read(self, path: str, size: int, offset: int, fh: int) -> bytes:
+        job, path_in_job = self._parse_path(path)
+        try:
+            file = job.get_output_file(path_in_job)
+        except BioLibError:
+            raise FuseOSError(errno.ENOENT) from None  # No such file or directory
+
+        return file.get_data(start=offset, length=size)
+
+    def release(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    def releasedir(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    def flush(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    @staticmethod
+    def _get_directory_attributes(timestamp_epoch_seconds: int) -> _AttributeDict:
+        return _AttributeDict(
+            st_atime=timestamp_epoch_seconds,
+            st_ctime=timestamp_epoch_seconds,
+            st_gid=os.getgid(),
+            st_mode=stat.S_IFDIR | 0o555,  # Directory that is readable and executable by owner, group, and others.
+            st_mtime=timestamp_epoch_seconds,
+            st_nlink=1,
+            st_size=1,
+            st_uid=os.getuid(),
+        )
+
+    @staticmethod
+    def _get_file_attributes(timestamp_epoch_seconds: int, size_in_bytes: int) -> _AttributeDict:
+        return _AttributeDict(
+            st_atime=timestamp_epoch_seconds,
+            st_ctime=timestamp_epoch_seconds,
+            st_gid=os.getgid(),
+            st_mode=stat.S_IFREG | 0o444,  # Regular file with read permissions for owner, group, and others.
+            st_mtime=timestamp_epoch_seconds,
+            st_nlink=1,
+            st_size=size_in_bytes,
+            st_uid=os.getuid(),
+        )
+
+    def _get_job_names_map(self, refresh_jobs=False) -> Dict[str, Job]:
+        current_time = time()
+        if not self._job_names_map or (current_time - self._jobs_last_fetched_at > 1 and refresh_jobs):
+            self._jobs_last_fetched_at = current_time
+            self._job_names_map = {job.get_name(): job for job in self._experiment.get_jobs(status='completed')}
+
+        return self._job_names_map
+
+    def _parse_path(self, path: str) -> Tuple[Job, str]:
+        path_splitted = path.split('/')
+        job_name = path_splitted[1]
+        path_in_job = '/' + '/'.join(path_splitted[2:])
+        job = self._get_job_names_map().get(job_name)
+        if not job:
+            raise FuseOSError(errno.ENOENT)  # No such file or directory
+
+        return job, path_in_job
+
+    # ----------------------------------- File system methods not implemented below -----------------------------------
+
+    def chmod(self, path, mode):
+        raise FuseOSError(errno.EACCES)
+
+    def chown(self, path, uid, gid):
+        raise FuseOSError(errno.EACCES)
+
+    def mknod(self, path, mode, dev):
+        raise FuseOSError(errno.EACCES)
+
+    def rmdir(self, path):
+        raise FuseOSError(errno.EACCES)
+
+    def mkdir(self, path, mode):
+        raise FuseOSError(errno.EACCES)
+
+    def unlink(self, path):
+        raise FuseOSError(errno.EACCES)
+
+    def symlink(self, target, source):
+        raise FuseOSError(errno.EACCES)
+
+    def rename(self, old, new):
+        raise FuseOSError(errno.EACCES)
+
+    def link(self, target, source):
+        raise FuseOSError(errno.EACCES)
+
+    def utimens(self, path, times=None):
+        raise FuseOSError(errno.EACCES)
+
+    def create(self, path, mode, fi=None):
+        raise FuseOSError(errno.EACCES)
+
+    def write(self, path, data, offset, fh):
+        raise FuseOSError(errno.EACCES)
+
+    def truncate(self, path, length, fh=None):
+        raise FuseOSError(errno.EACCES)
+
+    def fsync(self, path, datasync, fh):
+        raise FuseOSError(errno.EACCES)

biolib/_internal/http_client.py

@@ -1,5 +1,6 @@
 import json
 import platform
+import shutil
 import socket
 import ssl
 import subprocess
@@ -41,15 +42,24 @@ class HttpError(urllib.error.HTTPError):
 
 
 class HttpResponse:
-    def __init__(self, response) -> None:
+    def __init__(self, response, response_path) -> None:
         self.headers: Dict[str, str] = dict(response.headers)
         self.status_code: int = int(response.status)
-        self.content: bytes = response.read()
+        self.response_path = response_path
+        if self.response_path:
+            with open(self.response_path, 'wb') as out_file:
+                shutil.copyfileobj(response, out_file)
+        else:
+            self.content: bytes = response.read()
         self.url: str = response.geturl()
 
     @property
     def text(self) -> str:
-        return cast(str, self.content.decode('utf-8'))
+        if self.response_path:
+            with open(self.response_path, 'rb') as fp:
+                return cast(str, fp.read().decode('utf-8'))
+        else:
+            return cast(str, self.content.decode('utf-8'))
 
     def json(self):
         return json.loads(self.text)
@@ -66,6 +76,7 @@ class HttpClient:
         headers: Optional[Dict[str, str]] = None,
         retries: int = 5,
         timeout_in_seconds: Optional[int] = None,
+        response_path: Optional[str] = None,
     ) -> HttpResponse:
         if not HttpClient.ssl_context:
             HttpClient.ssl_context = _create_ssl_context()
@@ -83,7 +94,7 @@ class HttpClient:
         if timeout_in_seconds is None:
             timeout_in_seconds = 60 if isinstance(data, dict) else 180  # TODO: Calculate timeout based on data size
 
-        last_error: Optional[urllib.error.URLError] = None
+        last_error: Optional[Exception] = None
         for retry_count in range(retries + 1):
             if retry_count > 0:
                 time.sleep(5 * retry_count)
@@ -94,23 +105,34 @@ class HttpClient:
                     context=HttpClient.ssl_context,
                     timeout=timeout_in_seconds,
                 ) as response:
-                    return HttpResponse(response)
+                    return HttpResponse(response, response_path)
 
             except urllib.error.HTTPError as error:
-                if error.code == 502:
-                    logger_no_user_data.debug(f'HTTP {method} request failed with status 502 for "{url}"')
+                if error.code == 429:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 429 for "{url}"')
+                    last_error = error
+                elif error.code == 502:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 502 for "{url}"')
                     last_error = error
                 elif error.code == 503:
-                    logger_no_user_data.debug(f'HTTP {method} request failed with status 503 for "{url}"')
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 503 for "{url}"')
+                    last_error = error
+                elif error.code == 504:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 504 for "{url}"')
                     last_error = error
                 else:
                     raise HttpError(error) from None
 
             except urllib.error.URLError as error:
                 if isinstance(error.reason, socket.timeout):
-                    logger_no_user_data.debug(f'HTTP {method} request failed with read timeout for "{url}"')
+                    if retry_count > 0:
+                        logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
                     last_error = error
                 else:
                     raise error
+            except socket.timeout as error:
+                if retry_count > 0:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
+                last_error = error
 
         raise last_error or Exception(f'HTTP {method} request failed after {retries} retries for "{url}"')

biolib/_internal/lfs/__init__.py

@@ -0,0 +1 @@
+from .cache import prune_lfs_cache

biolib/_internal/libs/__init__.py

@@ -0,0 +1 @@
+# Note: this directory is purely for libraries to be directly included instead of as dependencies