ngio 0.2.0a2__py3-none-any.whl → 0.5.0b4__py3-none-any.whl

ngio/transforms/__init__.py

@@ -0,0 +1,5 @@
+"""Concrete IO transformations."""
+
+from ngio.transforms._zoom import ZoomTransform
+
+__all__ = ["ZoomTransform"]

ngio/transforms/_zoom.py

@@ -0,0 +1,19 @@
+from ngio.common._zoom import (
+    InterpolationOrder,
+)
+from ngio.images._abstract_image import AbstractImage
+from ngio.io_pipes._zoom_transform import BaseZoomTransform
+
+
+class ZoomTransform(BaseZoomTransform):
+    def __init__(
+        self,
+        input_image: AbstractImage,
+        target_image: AbstractImage,
+        order: InterpolationOrder = "nearest",
+    ) -> None:
+        super().__init__(
+            input_dimensions=input_image.dimensions,
+            target_dimensions=target_image.dimensions,
+            order=order,
+        )

ngio/utils/__init__.py

@@ -1,43 +1,45 @@
 """Various utilities for the ngio package."""
 
-import os
-
-from ngio.common._common_types import ArrayLike
-from ngio.utils._datasets import download_ome_zarr_dataset, list_ome_zarr_datasets
+from ngio.utils._datasets import (
+    download_ome_zarr_dataset,
+    list_ome_zarr_datasets,
+    print_datasets_infos,
+)
 from ngio.utils._errors import (
+    NgioError,
     NgioFileExistsError,
     NgioFileNotFoundError,
     NgioTableValidationError,
     NgioValidationError,
     NgioValueError,
 )
-from ngio.utils._logger import ngio_logger, set_logger_level
+from ngio.utils._fractal_fsspec_store import fractal_fsspec_store
 from ngio.utils._zarr_utils import (
     AccessModeLiteral,
+    NgioCache,
+    NgioSupportedStore,
     StoreOrGroup,
     ZarrGroupHandler,
+    copy_group,
     open_group_wrapper,
 )
 
-set_logger_level(os.getenv("NGIO_LOGGER_LEVEL", "WARNING"))
-
 __all__ = [
-    # Zarr
     "AccessModeLiteral",
-    "ArrayLike",
-    # Errors
+    "NgioCache",
+    "NgioError",
     "NgioFileExistsError",
     "NgioFileNotFoundError",
+    "NgioSupportedStore",
     "NgioTableValidationError",
     "NgioValidationError",
     "NgioValueError",
     "StoreOrGroup",
     "ZarrGroupHandler",
-    # Datasets
+    "copy_group",
     "download_ome_zarr_dataset",
+    "fractal_fsspec_store",
     "list_ome_zarr_datasets",
-    # Logger
-    "ngio_logger",
     "open_group_wrapper",
-    "set_logger_level",
+    "print_datasets_infos",
 ]

ngio/utils/_cache.py

@@ -0,0 +1,48 @@
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+class NgioCache(Generic[T]):
+    """A simple cache for NGIO objects."""
+
+    def __init__(self, use_cache: bool = True):
+        self._cache: dict[str, T] = {}
+        self._use_cache = use_cache
+
+    def _cache_sanity_check(self) -> None:
+        if len(self._cache) > 0:
+            raise RuntimeError(
+                "Cache is disabled, but cache contains items. "
+                "This indicates a logic error."
+            )
+
+    @property
+    def use_cache(self) -> bool:
+        return self._use_cache
+
+    @property
+    def cache(self) -> dict[str, T]:
+        return self._cache
+
+    @property
+    def is_empty(self) -> bool:
+        return len(self._cache) == 0
+
+    def get(self, key: str, default: T | None = None) -> T | None:
+        if not self._use_cache:
+            self._cache_sanity_check()
+            return default
+        return self._cache.get(key, default)
+
+    def set(self, key: str, value: T, overwrite: bool = True) -> None:
+        if not self._use_cache:
+            self._cache_sanity_check()
+            return
+        self._cache[key] = value
+
+    def clear(self) -> None:
+        if not self._use_cache:
+            self._cache_sanity_check()
+            return
+        self._cache.clear()

ngio/utils/_datasets.py

@@ -1,33 +1,129 @@
 """Download testing OME-Zarr datasets."""
 
+import shutil
 from pathlib import Path
+from typing import Literal
 
 import pooch
 
+from ngio.utils._errors import NgioValueError
+
+
+class UnzipAndRename(pooch.Unzip):
+    """Unzip and rename the extracted directory."""
+
+    def __init__(
+        self,
+        extract_dir: str = "",
+        out_name: str = "ome-zarr.zarr",
+        re_unzip: bool = True,
+        **kwargs,
+    ):
+        super().__init__(extract_dir=extract_dir, **kwargs)
+        self.out_name = out_name
+        self.re_unzip = re_unzip
+
+    def output_file(self) -> Path:
+        """Return the output file path."""
+        if self.extract_dir is None:
+            raise NgioValueError("extract_dir must be set for UnzipAndRename.")
+
+        return Path(self.extract_dir) / self.out_name
+
+    def _extract_file(self, fname, extract_dir):
+        """Extract the file and rename it."""
+        output_path = self.output_file()
+        if output_path.exists() and not self.re_unzip:
+            # Nothing to do, the file already exists and we are not re-unzipping
+            return None
+
+        tmp_dir = Path(extract_dir) / "tmp"
+        # If tmp_dir exists, remove it
+        if tmp_dir.exists():
+            shutil.rmtree(tmp_dir, ignore_errors=True)
+        tmp_dir.mkdir(parents=True, exist_ok=True)
+
+        super()._extract_file(fname, tmp_dir)
+
+        list_extracted_dirs = tmp_dir.iterdir()
+        # Keep only if ends with .zarr
+        list_extracted_dirs = filter(
+            lambda x: x.name.endswith(".zarr"),
+            list_extracted_dirs,
+        )
+        list_extracted_dirs = list(list_extracted_dirs)
+        if len(list_extracted_dirs) != 1:
+            raise NgioValueError(
+                "Expected one directory to be extracted, "
+                f"got {len(list_extracted_dirs)}."
+            )
+
+        extracted_dir = list_extracted_dirs[0]
+        if output_path.exists():
+            shutil.rmtree(output_path, ignore_errors=True)
+
+        extracted_dir.rename(output_path)
+        # Clean up the temporary directory
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
 _ome_zarr_zoo = {
     "CardiomyocyteTiny": {
         "url": "https://zenodo.org/records/13305156/files/20200812-CardiomyocyteDifferentiation14-Cycle1.zarr.zip",
         "known_hash": "md5:efc21fe8d4ea3abab76226d8c166452c",
-        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1.zarr.zip",
-        "processor": pooch.Unzip(extract_dir=""),
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1-tiny.zarr.zip",
+        "description": "Tiny cardiomyocyte dataset 3D (32MB).",
+    },
+    "CardiomyocyteTinyMip": {
+        "url": "https://zenodo.org/records/13305156/files/20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip",
+        "known_hash": "md5:51809479777cafbe9ac0f9fa5636aa95",
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1-tiny-mip.zarr.zip",
+        "description": "Tiny cardiomyocyte dataset 2D MIP (16.4MB).",
+    },
+    "CardiomyocyteSmall": {
+        "url": "https://zenodo.org/records/13305316/files/20200812-CardiomyocyteDifferentiation14-Cycle1.zarr.zip",
+        "known_hash": "md5:d5752ed4b72a9092a0290b3c04c0b9c2",
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1-small.zarr.zip",
+        "description": "Small cardiomyocyte dataset 3D (750MB).",
     },
     "CardiomyocyteSmallMip": {
         "url": "https://zenodo.org/records/13305316/files/20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip",
         "known_hash": "md5:3ed3ea898e0ed42d397da2e1dbe40750",
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1-small-mip.zarr.zip",
+        "description": "Small cardiomyocyte dataset 2D MIP (106MB).",
+    },
+    "CardiomyocyteMediumMip": {
+        "url": "https://zenodo.org/records/14826000/files/20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip",
+        "known_hash": "md5:3f932bbf7fc0577f58b97471707816a1",
         "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip",
-        "processor": pooch.Unzip(extract_dir=""),
+        "description": "Medium cardiomyocyte dataset 2D MIP (30GB).",
     },
 }
 
+AVAILABLE_DATASETS = Literal[
+    "CardiomyocyteTiny",
+    "CardiomyocyteTinyMip",
+    "CardiomyocyteSmall",
+    "CardiomyocyteSmallMip",
+    "CardiomyocyteMediumMip",
+]
+
 
 def list_ome_zarr_datasets() -> list[str]:
     """List available OME-Zarr datasets."""
     return list(_ome_zarr_zoo.keys())
 
 
+def print_datasets_infos() -> None:
+    for dataset_name, dataset_info in _ome_zarr_zoo.items():
+        print(f"{dataset_name} - Description: {dataset_info['description']}")
+
+
 def download_ome_zarr_dataset(
-    dataset_name: str,
-    download_dir: str = "data",
+    dataset_name: AVAILABLE_DATASETS | str,
+    download_dir: str | Path = "data",
+    re_unzip: bool = True,
+    progressbar: bool = False,
 ) -> Path:
     """Download an OME-Zarr dataset.
 
@@ -36,16 +132,28 @@ def download_ome_zarr_dataset(
     Args:
         dataset_name (str): The dataset name.
         download_dir (str): The download directory. Defaults to "data".
+        re_unzip (bool): If True, it will unzip the dataset even if it already exists.
+        progressbar (bool): If True, show a progress bar during download.
     """
     if dataset_name not in _ome_zarr_zoo:
-        raise ValueError(f"Dataset {dataset_name} not found in the OME-Zarr zoo.")
-    ome_zarr_url = _ome_zarr_zoo[dataset_name]
+        raise NgioValueError(f"Dataset {dataset_name} not found in the OME-Zarr zoo.")
+    zenodo_infos = _ome_zarr_zoo[dataset_name]
+
+    fname = zenodo_infos["fname"]
+    zarrname = fname.replace(".zip", "")
+
+    processor = UnzipAndRename(
+        extract_dir="",
+        out_name=zarrname,
+        re_unzip=re_unzip,
+    )
+
     pooch.retrieve(
+        url=zenodo_infos["url"],
+        known_hash=zenodo_infos["known_hash"],
+        fname=fname,
         path=download_dir,
-        **ome_zarr_url,
+        processor=processor,
+        progressbar=progressbar,
     )
-    path = Path(download_dir) / ome_zarr_url["fname"]
-
-    if isinstance(ome_zarr_url["processor"], pooch.Unzip):
-        path = path.with_suffix("")
-    return path
+    return processor.output_file()

ngio/utils/_fractal_fsspec_store.py

@@ -0,0 +1,42 @@
+import fsspec.implementations.http
+from aiohttp import ClientResponseError
+
+from ngio.utils import NgioValueError
+
+
+def fractal_fsspec_store(
+    url: str, fractal_token: str | None = None, client_kwargs: dict | None = None
+) -> fsspec.mapping.FSMap:
+    """Simple function to get an http fsspec store from a url."""
+    client_kwargs = {} if client_kwargs is None else client_kwargs
+    if fractal_token is not None:
+        client_kwargs["headers"] = {"Authorization": f"Bearer {fractal_token}"}
+    fs = fsspec.implementations.http.HTTPFileSystem(client_kwargs=client_kwargs)
+
+    store = fs.get_mapper(url)
+
+    possible_keys = [".zgroup", ".zarray"]
+    for key in possible_keys:
+        try:
+            value = store.get(key)
+            if value is not None:
+                break
+        except ClientResponseError as e:
+            if e.status == 401 and fractal_token is None:
+                raise NgioValueError(
+                    "No auto token is provided. You need a valid "
+                    f"'fractal_token' to access: {url}."
+                ) from e
+            elif e.status == 401 and fractal_token is not None:
+                raise NgioValueError(
+                    f"The 'fractal_token' provided is invalid for: {url}."
+                ) from e
+            else:
+                raise e
+    else:
+        raise NgioValueError(
+            f"Store {url} can not be read. Possible problems are: \n"
+            "- The url does not exist. \n"
+            "- The url is not a valid .zarr. \n"
+        )
+    return store