hoa-tools 1.0__py3-none-any.whl

hoa_tools/data/metadata/metadata/S-20-29_lung_left_VOI-05_6.5um_bm05.json

@@ -0,0 +1,205 @@
+{
+  "name": "S-20-29_lung_left_VOI-05_6.5um_bm05",
+  "dataset_type": "zoom",
+  "voi": "VOI-05",
+  "data": {
+    "shape": [3833, 3833, 20077],
+    "voxel_size_um": 6.5,
+    "gcs_url": "n5://gs://ucl-hip-ct-35a68e99feaae8932b1d44da0358940b/S-20-29/lung-left/6.5um_VOI-05_bm05/"
+  },
+  "sample": {
+    "organ": "lung",
+    "organ_context": "left",
+    "post_mortem_interval_hours": null,
+    "fixation_method": null,
+    "fixation_medium": null,
+    "organ_infilled": null,
+    "stabilisation_medium": null,
+    "degassing_method": null,
+    "scan_solvent": null,
+    "scan_solvent_concentration": null,
+    "scan_temperature": "room temperature"
+  },
+  "donor": {
+    "id": "S-20-29",
+    "age": 80,
+    "sex": "M",
+    "weight": 170.0,
+    "height": 90.0,
+    "cause_of_death": "Acute right heart failure with extensive Pulmonary artery thrombembolism, diffuse alveolar damage, shock kidneys bilateral with acute kidney failure, and acute liver failure COVID-19 disease",
+    "date_of_death": null,
+    "medical_history": "Kidney failure, arterial hypertension, Z.n. Spondylodesis LWK 2 and sacral vertebra 2, SARS-CoV2 infection, symptoms of memory loss and predementia, Dandy-Walker syndrome variant, post-mortem diagnostic.",
+    "diabetes": "Type 2",
+    "hypertension": null,
+    "smoker": null
+  },
+  "scan": {
+    "date": "2020-11-07",
+    "beamline": "BM05",
+    "energy": 95.0,
+    "current_start": 192.033,
+    "filling_mode": null,
+    "n_frames": 6000,
+    "n_ref": 3,
+    "n_dark": 0,
+    "latency_time": 0.0015,
+    "exposure_time": 0.015,
+    "subframe_time": null,
+    "n_subframes": 4,
+    "scan_type": "zseries",
+    "scan_range": 360.0,
+    "n_scans": 59,
+    "acquisition": "half",
+    "z_step": null,
+    "scan_time": 420.0,
+    "filters": ["Mo 0.1mm", "SiO2 50mm rods 10x5mm"],
+    "scintillator": "LuAG:Ce 1000um",
+    "optic": "Zoom",
+    "optic_magnification": 0.941,
+    "distance_source_sample": null,
+    "distance_sample_detector": 500.0,
+    "sensor_name": "PCO edge 4.2 CLHS",
+    "sensor_mode": "rolling shutter",
+    "sensor_roi_x_size": 2043,
+    "sensor_roi_y_size": 20077,
+    "sensor_binning": null,
+    "pixel_size": 6.5,
+    "xray_magnification": null,
+    "technique": "Hierarchical Phase-Contrast Tomography (HiP-CT)",
+    "experiment_type": "Tomography"
+  },
+  "proposal": {
+    "proposal_number": "md1252",
+    "title": "Multiscale Quantification of Covid-19's impact on lung vasculature from whole lobe to alveolar/microvascular scales",
+    "proposers": [
+      "Peter Lee",
+      "Paul Tafforeau",
+      "Danny Jonigk",
+      "Maximilian Ackermann",
+      "Mark P. Kuhnel",
+      "Elodie Boller",
+      "Willi Wagner"
+    ]
+  },
+  "registration": {
+    "source_dataset": "S-20-29_lung_left_VOI-05_6.5um_bm05",
+    "target_dataset": "S-20-29_lung_left_complete-organ_25.31um_bm05",
+    "translation": [1257.7257673685622, 1005.485226013251, 789.8548884643609],
+    "rotation": -5.299835000447964,
+    "scale": 0.2568154947103331
+  },
+  "citation": {
+    "title": "Zoom (VOI-05) at 6.5um of the lung of donor S-20-29, scanned at ESRF on beamline BM05.",
+    "contributors": [
+      {
+        "first_name": "The",
+        "last_name": "Human Organ Atlas Collaboration",
+        "orcid": null,
+        "roles": []
+      },
+      {
+        "first_name": "Claire",
+        "last_name": "Walsh",
+        "orcid": "0000-0003-3769-3392",
+        "roles": ["Conceptualization", "Funding acquisition"]
+      },
+      {
+        "first_name": "Danny",
+        "last_name": "Jonigk",
+        "orcid": "0000-0002-5251-2281",
+        "roles": ["Funding acquisition"]
+      },
+      {
+        "first_name": "David",
+        "last_name": "Stansby",
+        "orcid": "0000-0002-1365-1908",
+        "roles": ["Data curation", "Software"]
+      },
+      {
+        "first_name": "Elodie",
+        "last_name": "Boller",
+        "orcid": null,
+        "roles": ["Funding acquisition"]
+      },
+      {
+        "first_name": "Guillaume",
+        "last_name": "Gaisne",
+        "orcid": "0000-0002-3401-7930",
+        "roles": ["Data curation", "Software"]
+      },
+      {
+        "first_name": "Hector",
+        "last_name": "Dejea I Velardo",
+        "orcid": "0000-0003-2584-9812",
+        "roles": ["Data curation", "Investigation"]
+      },
+      {
+        "first_name": "Joanna",
+        "last_name": "Purzycka",
+        "orcid": "0000-0001-8742-5279",
+        "roles": ["Data curation"]
+      },
+      {
+        "first_name": "Joseph",
+        "last_name": "Brunet",
+        "orcid": "0000-0002-8424-9510",
+        "roles": ["Data curation", "Investigation", "Software"]
+      },
+      {
+        "first_name": "Mark",
+        "last_name": "P. Kuhnel",
+        "orcid": null,
+        "roles": ["Funding acquisition"]
+      },
+      {
+        "first_name": "Maximilian",
+        "last_name": "Ackermann",
+        "orcid": "0000-0001-9996-2477",
+        "roles": ["Funding acquisition"]
+      },
+      {
+        "first_name": "Paul",
+        "last_name": "Tafforeau",
+        "orcid": "0000-0002-5962-1683",
+        "roles": ["Conceptualization", "Data curation", "Funding acquisition"]
+      },
+      {
+        "first_name": "Peter",
+        "last_name": "Lee",
+        "orcid": "0000-0002-3898-8881",
+        "roles": ["Conceptualization", "Funding acquisition"]
+      },
+      {
+        "first_name": "Theresa",
+        "last_name": "Urban",
+        "orcid": "0000-0002-0352-8180",
+        "roles": ["Data curation", "Investigation", "Software"]
+      },
+      {
+        "first_name": "Willi",
+        "last_name": "Wagner",
+        "orcid": null,
+        "roles": ["Funding acquisition"]
+      }
+    ],
+    "doi": "10.15151/ESRF-DC-1773964041",
+    "author_list": [
+      "The Human Organ Atlas Collaboration",
+      "Claire Walsh",
+      "Danny Jonigk",
+      "David Stansby",
+      "Elodie Boller",
+      "Guillaume Gaisne",
+      "Hector Dejea I Velardo",
+      "Joanna Purzycka",
+      "Joseph Brunet",
+      "Mark P. Kuhnel",
+      "Maximilian Ackermann",
+      "Paul Tafforeau",
+      "Peter Lee",
+      "Theresa Urban",
+      "Willi Wagner"
+    ]
+  },
+  "metadata_version": "0.9"
+}

hoa_tools/data/metadata/metadata/S-20-29_lung_left_complete-organ_25.31um_bm05.json

@@ -0,0 +1,199 @@
+{
+  "name": "S-20-29_lung_left_complete-organ_25.31um_bm05",
+  "dataset_type": "overview",
+  "voi": "complete-organ",
+  "data": {
+    "shape": [3814, 5350, 6391],
+    "voxel_size_um": 25.31,
+    "gcs_url": "n5://gs://ucl-hip-ct-35a68e99feaae8932b1d44da0358940b/S-20-29/lung-left/25.31um_complete-organ_bm05/"
+  },
+  "sample": {
+    "organ": "lung",
+    "organ_context": "left",
+    "post_mortem_interval_hours": null,
+    "fixation_method": null,
+    "fixation_medium": null,
+    "organ_infilled": null,
+    "stabilisation_medium": null,
+    "degassing_method": null,
+    "scan_solvent": null,
+    "scan_solvent_concentration": null,
+    "scan_temperature": "room temperature"
+  },
+  "donor": {
+    "id": "S-20-29",
+    "age": 80,
+    "sex": "M",
+    "weight": 170.0,
+    "height": 90.0,
+    "cause_of_death": "Acute right heart failure with extensive Pulmonary artery thrombembolism, diffuse alveolar damage, shock kidneys bilateral with acute kidney failure, and acute liver failure COVID-19 disease",
+    "date_of_death": null,
+    "medical_history": "Kidney failure, arterial hypertension, Z.n. Spondylodesis LWK 2 and sacral vertebra 2, SARS-CoV2 infection, symptoms of memory loss and predementia, Dandy-Walker syndrome variant, post-mortem diagnostic.",
+    "diabetes": "Type 2",
+    "hypertension": null,
+    "smoker": null
+  },
+  "scan": {
+    "date": "2020-10-27",
+    "beamline": "BM05",
+    "energy": 96.0,
+    "current_start": 190.786,
+    "filling_mode": null,
+    "n_frames": 9990,
+    "n_ref": 3,
+    "n_dark": 0,
+    "latency_time": 0.0007,
+    "exposure_time": 0.007,
+    "subframe_time": null,
+    "n_subframes": 4,
+    "scan_type": "zseries",
+    "scan_range": 360.0,
+    "n_scans": null,
+    "acquisition": "quarter",
+    "z_step": null,
+    "scan_time": 330.0,
+    "filters": ["Mo 0.1mm", "SiO2 50mm rods 10x5mm"],
+    "scintillator": "LuAG:Ce 1000um",
+    "optic": "Zoom",
+    "optic_magnification": 2.154,
+    "distance_source_sample": null,
+    "distance_sample_detector": 3650.0,
+    "sensor_name": "PCO edge 4.2 CLHS",
+    "sensor_mode": "rolling shutter",
+    "sensor_roi_x_size": null,
+    "sensor_roi_y_size": null,
+    "sensor_binning": null,
+    "pixel_size": 25.31,
+    "xray_magnification": null,
+    "technique": "Hierarchical Phase-Contrast Tomography (HiP-CT)",
+    "experiment_type": "Tomography"
+  },
+  "proposal": {
+    "proposal_number": "md1252",
+    "title": "Multiscale Quantification of Covid-19's impact on lung vasculature from whole lobe to alveolar/microvascular scales",
+    "proposers": [
+      "Peter Lee",
+      "Paul Tafforeau",
+      "Danny Jonigk",
+      "Maximilian Ackermann",
+      "Mark P. Kuhnel",
+      "Elodie Boller",
+      "Willi Wagner"
+    ]
+  },
+  "registration": null,
+  "citation": {
+    "title": "Overview at 25.31um of the lung of donor S-20-29, scanned at ESRF on beamline BM05.",
+    "contributors": [
+      {
+        "first_name": "The",
+        "last_name": "Human Organ Atlas Collaboration",
+        "orcid": null,
+        "roles": []
+      },
+      {
+        "first_name": "Claire",
+        "last_name": "Walsh",
+        "orcid": "0000-0003-3769-3392",
+        "roles": ["Conceptualization", "Funding acquisition"]
+      },
+      {
+        "first_name": "Danny",
+        "last_name": "Jonigk",
+        "orcid": "0000-0002-5251-2281",
+        "roles": ["Funding acquisition"]
+      },
+      {
+        "first_name": "David",
+        "last_name": "Stansby",
+        "orcid": "0000-0002-1365-1908",
+        "roles": ["Data curation", "Software"]
+      },
+      {
+        "first_name": "Elodie",
+        "last_name": "Boller",
+        "orcid": null,
+        "roles": ["Funding acquisition"]
+      },
+      {
+        "first_name": "Guillaume",
+        "last_name": "Gaisne",
+        "orcid": "0000-0002-3401-7930",
+        "roles": ["Data curation", "Software"]
+      },
+      {
+        "first_name": "Hector",
+        "last_name": "Dejea I Velardo",
+        "orcid": "0000-0003-2584-9812",
+        "roles": ["Data curation", "Investigation"]
+      },
+      {
+        "first_name": "Joanna",
+        "last_name": "Purzycka",
+        "orcid": "0000-0001-8742-5279",
+        "roles": ["Data curation"]
+      },
+      {
+        "first_name": "Joseph",
+        "last_name": "Brunet",
+        "orcid": "0000-0002-8424-9510",
+        "roles": ["Data curation", "Investigation", "Software"]
+      },
+      {
+        "first_name": "Mark",
+        "last_name": "P. Kuhnel",
+        "orcid": null,
+        "roles": ["Funding acquisition"]
+      },
+      {
+        "first_name": "Maximilian",
+        "last_name": "Ackermann",
+        "orcid": "0000-0001-9996-2477",
+        "roles": ["Funding acquisition"]
+      },
+      {
+        "first_name": "Paul",
+        "last_name": "Tafforeau",
+        "orcid": "0000-0002-5962-1683",
+        "roles": ["Conceptualization", "Data curation", "Funding acquisition"]
+      },
+      {
+        "first_name": "Peter",
+        "last_name": "Lee",
+        "orcid": "0000-0002-3898-8881",
+        "roles": ["Conceptualization", "Funding acquisition"]
+      },
+      {
+        "first_name": "Theresa",
+        "last_name": "Urban",
+        "orcid": "0000-0002-0352-8180",
+        "roles": ["Data curation", "Investigation", "Software"]
+      },
+      {
+        "first_name": "Willi",
+        "last_name": "Wagner",
+        "orcid": null,
+        "roles": ["Funding acquisition"]
+      }
+    ],
+    "doi": "10.15151/ESRF-DC-1773966447",
+    "author_list": [
+      "The Human Organ Atlas Collaboration",
+      "Claire Walsh",
+      "Danny Jonigk",
+      "David Stansby",
+      "Elodie Boller",
+      "Guillaume Gaisne",
+      "Hector Dejea I Velardo",
+      "Joanna Purzycka",
+      "Joseph Brunet",
+      "Mark P. Kuhnel",
+      "Maximilian Ackermann",
+      "Paul Tafforeau",
+      "Peter Lee",
+      "Theresa Urban",
+      "Willi Wagner"
+    ]
+  },
+  "metadata_version": "0.9"
+}

hoa_tools/dataset.py

@@ -0,0 +1,229 @@
+"""
+Tools for working with individual datasets.
+
+The core class representing a single dataset is [`Dataset`][hoa_tools.dataset.Dataset].
+This inherits from [`HOAMetadata`][hoa_tools.metadata.HOAMetadata],
+which stores all the metadata for a given dataset.
+
+[`Dataset`][hoa_tools.dataset.Dataset] objects are not designed to be created by users.
+To get a [`Dataset`][hoa_tools.dataset.Dataset], use the
+[`get_dataset`][hoa_tools.dataset.get_dataset] function in this module.
+"""
+
+from functools import cached_property
+from pathlib import Path
+from typing import Literal
+
+import dask.array.core
+import gcsfs
+import numpy as np
+import xarray as xr
+import zarr.core
+import zarr.n5
+import zarr.storage
+
+from hoa_tools._n5 import N5FSStore
+from hoa_tools.metadata import HOAMetadata
+
+__all__ = ["Dataset", "get_dataset"]
+
+
+class Dataset(HOAMetadata):
+    """
+    An individual Human Organ Atlas dataset.
+    """
+
+    def __str__(self) -> str:
+        return f"Dataset(name={self.name})"
+
+    def _organ_str(self) -> str:
+        """
+        Get name of organ, with organ context appended if present.
+        """
+        organ_str = str(self.sample.organ)
+        if self.sample.organ_context:
+            organ_str += "_" + self.sample.organ_context
+        return organ_str
+
+    def _voxel_size_str(self) -> str:
+        # Make sure that voxel size includes a .0 if an integer value.
+        return str(float(self.data.voxel_size_um))
+
+    @property
+    def is_full_organ(self) -> bool:
+        """
+        Whether this dataset contains the whole organ or not.
+        """
+        return self.voi.startswith("complete")
+
+    @property
+    def is_zoom(self) -> bool:
+        """
+        Whether this dataset is a zoom or not.
+        """
+        return not self.is_full_organ
+
+    def get_children(self) -> list["Dataset"]:
+        """
+        Get child dataset(s).
+
+        Child datasets are high-resolution zooms of a full-organ dataset.
+
+        Notes
+        -----
+        For full-organ datasets, this returns an empty list.
+
+        """
+        children = [
+            d
+            for d in _DATASETS.values()
+            if (
+                d.donor.id == self.donor.id
+                and d.sample.organ == self.sample.organ
+                and d.scan.beamline == self.scan.beamline
+                and d.is_zoom
+            )
+        ]
+        return sorted(children, key=lambda c: c.name)
+
+    def get_parents(self) -> list["Dataset"]:
+        """
+        Get parent dataset(s).
+
+        Parent datasets are full-organ datasets from which a zoom dataset is taken from.
+
+        Notes
+        -----
+        For zoom datasets, this returns an empty list.
+
+        """
+        parents = [
+            d
+            for d in _DATASETS.values()
+            if (
+                d.donor.id == self.donor.id
+                and d.sample.organ == self.sample.organ
+                and d.scan.beamline == self.scan.beamline
+                and d.is_full_organ
+            )
+        ]
+        return sorted(parents, key=lambda c: c.name)
+
+    @property
+    def _remote_fmt(self) -> Literal["n5", "zarr"]:
+        if self.data.gcs_url.startswith("n5://"):
+            return "n5"
+        if self.data.gcs_url.startswith("zarr://"):
+            return "zarr"
+        raise RuntimeError("URL must start with n5:// or zarr://")
+
+    @cached_property
+    def _remote_store(self) -> zarr.Group:
+        """
+        Remote data store.
+        """
+        gcs_url = self.data.gcs_url
+        gcs_path = gcs_url.removeprefix("n5://gs://").removeprefix("zarr://gs://")
+
+        # n5://gs://ucl-hip-ct-35a68e99feaae8932b1d44da0358940b/S-20-29/heart/2.5um_VOI-01_bm05/
+        bucket, path = gcs_path.split("/", maxsplit=1)
+        fs = gcsfs.GCSFileSystem(project=bucket, token="anon", access="read_only")  # noqa: S106
+        if self._remote_fmt == "n5":
+            store = N5FSStore(url=bucket, fs=fs, mode="r")
+        elif self._remote_fmt == "zarr":
+            store = zarr.storage.FSStore(url=bucket, fs=fs, mode="r")
+
+        return zarr.open_group(store, mode="r", path=path)
+
+    def _remote_array(self, *, downsample_level: int) -> zarr.core.Array:
+        """
+        Get an object representing the data array in the remote Google Cloud Store.
+        """
+        if not downsample_level >= 0:
+            msg = "level must be >= 0"
+            raise ValueError(msg)
+
+        if self._remote_fmt == "n5":
+            key = f"s{downsample_level}"
+        else:
+            key = f"{downsample_level}"
+        return self._remote_store[key]
+
+    def data_array(self, *, downsample_level: int) -> xr.DataArray:
+        """
+        Get a DataArray representing the array for this image.
+        """
+        remote_array = self._remote_array(downsample_level=downsample_level)
+        dask_array = dask.array.core.from_array(  # type: ignore[no-untyped-call]
+            remote_array, chunks=remote_array.chunks
+        )
+        if self._remote_fmt == "zarr":
+            dask_array = dask_array.T
+
+        spacing = self.data.voxel_size_um * 2**downsample_level
+        return xr.DataArray(
+            dask_array,
+            name=self.name,
+            dims=["z", "y", "x"],
+            coords={
+                "z": xr.DataArray(
+                    data=(np.arange(dask_array.shape[0]) * spacing),
+                    dims=["z"],
+                    attrs={"units": "μm"},
+                ),
+                "y": xr.DataArray(
+                    data=(np.arange(dask_array.shape[1]) * spacing),
+                    dims=["y"],
+                    attrs={"units": "μm"},
+                ),
+                "x": xr.DataArray(
+                    data=(np.arange(dask_array.shape[2]) * spacing),
+                    dims=["x"],
+                    attrs={"units": "μm"},
+                ),
+            },
+        )
+
+
+def _get_datasets(data_dir: Path) -> dict[str, Dataset]:
+    """
+    Load dataset metadtata files.
+    """
+    return {
+        f.stem: Dataset.model_validate_json(f.read_text())
+        for f in (data_dir).glob("*.json")
+    }
+
+
+_META_DIR = Path(__file__).parent / "data" / "metadata" / "metadata"
+_DATASETS = _get_datasets(_META_DIR)
+if len(_DATASETS) == 0:
+    raise ImportError(
+        "Did not find any dataset metadata files. "
+        "This means there is something wrong with the hoa-tools installation. "
+        "If you are installing from source, you might need to "
+        "initialise and fetch git submodules. "
+    )
+
+
+def get_dataset(name: str) -> Dataset:
+    """
+    Get a dataset from its name.
+
+    The name of datasets can be looked up using the `hoa_tools.inventory` module.
+    """
+    return _DATASETS[name]
+
+
+def change_metadata_directory(data_dir: Path) -> None:
+    """
+    Update available datasets from another directory of metadata files.
+
+    Designed for internal project members to load metadata files that aren't yet public.
+    """
+    global _DATASETS  # noqa: PLW0603
+    _DATASETS = _get_datasets(data_dir)
+    if len(_DATASETS) == 0:
+        raise FileNotFoundError(
+            f"Did not find any dataset metadata files at {data_dir}"  # noqa: EM102
+        )

hoa_tools/inventory.py

@@ -0,0 +1,30 @@
+"""
+Tools for working with the dataset inventory.
+"""
+
+import pandas as pd
+
+from hoa_tools.dataset import _DATASETS
+
+
+def load_inventory() -> pd.DataFrame:
+    """
+    Load the dataset inventory.
+
+    Returns
+    -------
+    inventory :
+        Dataset inventory.
+
+    """
+    data = [
+        {
+            "donor": d.donor.id,
+            "organ": d.sample.organ,
+            "organ_context": d.sample.organ_context,
+            "voi": d.voi,
+            "voxel_size_um": d.data.voxel_size_um,
+        }
+        for d in _DATASETS.values()
+    ]
+    return pd.DataFrame(data=data, index=list(_DATASETS.keys()))