ngio 0.5.0b6__py3-none-any.whl

ngio/utils/_datasets.py

@@ -0,0 +1,165 @@
+"""Download testing OME-Zarr datasets."""
+
+import shutil
+from pathlib import Path
+from typing import Literal
+
+import pooch
+
+from ngio.utils._errors import NgioValueError
+
+
+class UnzipAndRename(pooch.Unzip):
+    """Unzip and rename the extracted directory."""
+
+    def __init__(
+        self,
+        extract_dir: str = "",
+        out_name: str = "ome-zarr.zarr",
+        re_unzip: bool = True,
+        **kwargs,
+    ):
+        super().__init__(extract_dir=extract_dir, **kwargs)
+        self.out_name = out_name
+        self.re_unzip = re_unzip
+
+    def output_file(self) -> Path:
+        """Return the output file path."""
+        if self.extract_dir is None:
+            raise NgioValueError("extract_dir must be set for UnzipAndRename.")
+
+        return Path(self.extract_dir) / self.out_name
+
+    def _extract_file(self, fname, extract_dir):
+        """Extract the file and rename it."""
+        output_path = self.output_file()
+        if output_path.exists() and not self.re_unzip:
+            # Nothing to do, the file already exists and we are not re-unzipping
+            return None
+
+        tmp_dir = Path(extract_dir) / "tmp"
+        # If tmp_dir exists, remove it
+        if tmp_dir.exists():
+            shutil.rmtree(tmp_dir, ignore_errors=True)
+        tmp_dir.mkdir(parents=True, exist_ok=True)
+
+        super()._extract_file(fname, tmp_dir)
+
+        list_extracted_dirs = tmp_dir.iterdir()
+        # Keep only if ends with .zarr
+        list_extracted_dirs = filter(
+            lambda x: x.name.endswith(".zarr"),
+            list_extracted_dirs,
+        )
+        list_extracted_dirs = list(list_extracted_dirs)
+        if len(list_extracted_dirs) != 1:
+            raise NgioValueError(
+                "Expected one directory to be extracted, "
+                f"got {len(list_extracted_dirs)}."
+            )
+
+        extracted_dir = list_extracted_dirs[0]
+        if output_path.exists():
+            shutil.rmtree(output_path, ignore_errors=True)
+
+        extracted_dir.rename(output_path)
+        # Clean up the temporary directory
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+_ome_zarr_zoo = {
+    "CardiomyocyteTiny": {
+        "url": "https://zenodo.org/records/13305156/files/20200812-CardiomyocyteDifferentiation14-Cycle1.zarr.zip",
+        "known_hash": "md5:efc21fe8d4ea3abab76226d8c166452c",
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1-tiny.zarr.zip",
+        "description": "Tiny cardiomyocyte dataset 3D (32MB).",
+    },
+    "CardiomyocyteTinyMip": {
+        "url": "https://zenodo.org/records/13305156/files/20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip",
+        "known_hash": "md5:51809479777cafbe9ac0f9fa5636aa95",
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1-tiny-mip.zarr.zip",
+        "description": "Tiny cardiomyocyte dataset 2D MIP (16.4MB).",
+    },
+    "CardiomyocyteSmall": {
+        "url": "https://zenodo.org/records/13305316/files/20200812-CardiomyocyteDifferentiation14-Cycle1.zarr.zip",
+        "known_hash": "md5:d5752ed4b72a9092a0290b3c04c0b9c2",
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1-small.zarr.zip",
+        "description": "Small cardiomyocyte dataset 3D (750MB).",
+    },
+    "CardiomyocyteSmallMip": {
+        "url": "https://zenodo.org/records/13305316/files/20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip",
+        "known_hash": "md5:3ed3ea898e0ed42d397da2e1dbe40750",
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1-small-mip.zarr.zip",
+        "description": "Small cardiomyocyte dataset 2D MIP (106MB).",
+    },
+    "CardiomyocyteMediumMip": {
+        "url": "https://zenodo.org/records/14826000/files/20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip",
+        "known_hash": "md5:3f932bbf7fc0577f58b97471707816a1",
+        "fname": "20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip",
+        "description": "Medium cardiomyocyte dataset 2D MIP (30GB).",
+    },
+}
+
+AVAILABLE_DATASETS = Literal[
+    "CardiomyocyteTiny",
+    "CardiomyocyteTinyMip",
+    "CardiomyocyteSmall",
+    "CardiomyocyteSmallMip",
+    "CardiomyocyteMediumMip",
+]
+
+
+def list_ome_zarr_datasets() -> list[str]:
+    """List available OME-Zarr datasets."""
+    return list(_ome_zarr_zoo.keys())
+
+
+def print_datasets_infos() -> None:
+    for dataset_name, dataset_info in _ome_zarr_zoo.items():
+        print(f"{dataset_name} - Description: {dataset_info['description']}")
+
+
+def download_ome_zarr_dataset(
+    dataset_name: AVAILABLE_DATASETS | str,
+    download_dir: str | Path = "data",
+    re_unzip: bool = True,
+    progressbar: bool = False,
+) -> Path:
+    """Download an OME-Zarr dataset.
+
+    To list available datasets, use `list_ome_zarr_datasets`.
+
+    Args:
+        dataset_name (str): The dataset name.
+        download_dir (str): The download directory. Defaults to "data".
+        re_unzip (bool): If True, it will unzip the dataset even if it already exists.
+        progressbar (bool): If True, show a progress bar during download.
+    """
+    if dataset_name not in _ome_zarr_zoo:
+        raise NgioValueError(f"Dataset {dataset_name} not found in the OME-Zarr zoo.")
+    zenodo_infos = _ome_zarr_zoo[dataset_name]
+
+    fname = zenodo_infos["fname"]
+    zarrname = fname.replace(".zip", "")
+
+    processor = UnzipAndRename(
+        extract_dir="",
+        out_name=zarrname,
+        re_unzip=re_unzip,
+    )
+
+    pooch.retrieve(
+        url=zenodo_infos["url"],
+        known_hash=zenodo_infos["known_hash"],
+        fname=fname,
+        path=download_dir,
+        processor=processor,
+        progressbar=progressbar,
+        # Add User-Agent to avoid 403 errors from Zenodo
+        downloader=pooch.HTTPDownloader(
+            headers={
+                "User-Agent": f"pooch/{pooch.__version__} (https://github.com/BioVisionCenter/ngio)"
+            }
+        ),
+    )
+    return processor.output_file()

ngio/utils/_errors.py

@@ -0,0 +1,37 @@
+# Create a generic error class for the NGFF project
+
+
+class NgioError(Exception):
+    """Base class for all errors in the NGFF project."""
+
+    pass
+
+
+class NgioFileNotFoundError(NgioError, FileNotFoundError):
+    """Error raised when a file is not found."""
+
+    pass
+
+
+class NgioFileExistsError(NgioError, FileExistsError):
+    """Error raised when a file already exists."""
+
+    pass
+
+
+class NgioValidationError(NgioError, ValueError):
+    """Generic error raised when a file does not pass validation."""
+
+    pass
+
+
+class NgioTableValidationError(NgioError):
+    """Error raised when a table does not pass validation."""
+
+    pass
+
+
+class NgioValueError(NgioError, ValueError):
+    """Error raised when a value does not pass a run time test."""
+
+    pass

ngio/utils/_fractal_fsspec_store.py

@@ -0,0 +1,42 @@
+import fsspec.implementations.http
+from aiohttp import ClientResponseError
+
+from ngio.utils import NgioValueError
+
+
+def fractal_fsspec_store(
+    url: str, fractal_token: str | None = None, client_kwargs: dict | None = None
+) -> fsspec.mapping.FSMap:
+    """Simple function to get an http fsspec store from a url."""
+    client_kwargs = {} if client_kwargs is None else client_kwargs
+    if fractal_token is not None:
+        client_kwargs["headers"] = {"Authorization": f"Bearer {fractal_token}"}
+    fs = fsspec.implementations.http.HTTPFileSystem(client_kwargs=client_kwargs)
+
+    store = fs.get_mapper(url)
+
+    possible_keys = [".zgroup", ".zarray"]
+    for key in possible_keys:
+        try:
+            value = store.get(key)
+            if value is not None:
+                break
+        except ClientResponseError as e:
+            if e.status == 401 and fractal_token is None:
+                raise NgioValueError(
+                    "No auto token is provided. You need a valid "
+                    f"'fractal_token' to access: {url}."
+                ) from e
+            elif e.status == 401 and fractal_token is not None:
+                raise NgioValueError(
+                    f"The 'fractal_token' provided is invalid for: {url}."
+                ) from e
+            else:
+                raise e
+    else:
+        raise NgioValueError(
+            f"Store {url} can not be read. Possible problems are: \n"
+            "- The url does not exist. \n"
+            "- The url is not a valid .zarr. \n"
+        )
+    return store