ngio 0.2.0a2__py3-none-any.whl → 0.2.0b1__py3-none-any.whl

ngio/tables/backends/_anndata_v1.py

@@ -10,6 +10,7 @@ from ngio.tables.backends._anndata_utils import (
     custom_read_zarr,
     dataframe_to_anndata,
 )
+from ngio.utils import NgioValueError
 
 
 class AnnDataBackend(AbstractTableBackend):
@@ -65,7 +66,7 @@ class AnnDataBackend(AbstractTableBackend):
         """Consolidate the metadata in the store."""
         store = self._group_handler.store
         if not isinstance(store, str | Path):
-            raise ValueError(
+            raise NgioValueError(
                 "To write an AnnData object the store must be a local path/str."
             )
 

ngio/tables/backends/_json_v1.py

@@ -13,7 +13,7 @@ class JsonTableBackend(AbstractTableBackend):
     @staticmethod
     def backend_name() -> str:
         """The name of the backend."""
-        return "json_v1"
+        return "experimental_json_v1"
 
     @staticmethod
     def implements_anndata() -> bool:

ngio/tables/tables_container.py

@@ -4,6 +4,7 @@ from typing import Literal, Protocol
 
 from ngio.tables.v1 import FeatureTableV1, MaskingROITableV1, RoiTableV1
 from ngio.tables.v1._generic_table import GenericTable
+from ngio.tables.v1._roi_table import _GenericRoiTableV1
 from ngio.utils import (
     AccessModeLiteral,
     NgioValidationError,
@@ -12,6 +13,7 @@ from ngio.utils import (
     ZarrGroupHandler,
 )
 
+GenericRoiTable = _GenericRoiTableV1
 RoiTable = RoiTableV1
 MaskingROITable = MaskingROITableV1
 FeatureTable = FeatureTableV1
@@ -55,7 +57,9 @@ class Table(Protocol):
         ...
 
 
-TypedTable = Literal["roi_table", "masking_roi_table", "feature_table"]
+TypedTable = Literal[
+    "roi_table", "masking_roi_table", "feature_table", "generic_roi_table"
+]
 
 
 def _unique_table_name(type_name, version) -> str:
@@ -99,7 +103,6 @@ class ImplementedTables:
                 return table
             except Exception as e:
                 _errors[name] = e
-        print(_errors)
         # If no table was found, we can try to load the table from a generic table
         try:
             table = GenericTable._from_handler(
@@ -183,6 +186,13 @@ class TablesContainer:
         handler = self._group_handler.derive_handler(path=name)
         return handler
 
+    def list_roi_tables(self) -> list[str]:
+        """List all ROI tables in the group."""
+        _tables = []
+        for _type in ["roi_table", "masking_roi_table"]:
+            _tables.extend(self.list(_type))
+        return _tables
+
     def list(self, filter_types: str | None = None) -> list[str]:
         """List all labels in the group."""
         tables = self._get_tables_list()

ngio/tables/v1/__init__.py

@@ -2,7 +2,6 @@
 
 from ngio.tables.v1._feature_table import FeatureTableV1
 from ngio.tables.v1._generic_table import GenericTable
-from ngio.tables.v1._masking_roi_table import MaskingROITableV1
-from ngio.tables.v1._roi_table import RoiTableV1
+from ngio.tables.v1._roi_table import MaskingROITableV1, RoiTableV1
 
 __all__ = ["FeatureTableV1", "GenericTable", "MaskingROITableV1", "RoiTableV1"]

ngio/tables/v1/_feature_table.py

@@ -11,7 +11,7 @@ from pydantic import BaseModel
 
 from ngio.tables._validators import validate_index_key
 from ngio.tables.backends import ImplementedTableBackends
-from ngio.utils import ZarrGroupHandler
+from ngio.utils import NgioValueError, ZarrGroupHandler
 
 
 class RegionMeta(BaseModel):
@@ -83,7 +83,7 @@ class FeatureTableV1:
     def dataframe(self) -> pd.DataFrame:
         """Return the table as a DataFrame."""
         if self._dataframe is None and self._table_backend is None:
-            raise ValueError(
+            raise NgioValueError(
                 "The table does not have a DataFrame in memory nor a backend."
             )
 
@@ -91,7 +91,7 @@ class FeatureTableV1:
             self._dataframe = self._table_backend.load_as_dataframe()
 
         if self._dataframe is None:
-            raise ValueError(
+            raise NgioValueError(
                 "The table does not have a DataFrame in memory nor a backend."
             )
         return self._dataframe
@@ -125,7 +125,9 @@ class FeatureTableV1:
             meta.backend = backend_name
 
         if not backend.implements_dataframe:
-            raise ValueError("The backend does not implement the dataframe protocol.")
+            raise NgioValueError(
+                "The backend does not implement the dataframe protocol."
+            )
 
         table = cls()
         table._meta = meta
@@ -151,7 +153,7 @@ class FeatureTableV1:
     def consolidate(self) -> None:
         """Write the current state of the table to the Zarr file."""
         if self._table_backend is None:
-            raise ValueError(
+            raise NgioValueError(
                 "No backend set for the table. "
                 "Please add the table to a OME-Zarr Image before calling consolidate."
             )

ngio/tables/v1/_generic_table.py

@@ -1,10 +1,15 @@
 """Implementation of a generic table class."""
 
 import pandas as pd
+from anndata import AnnData
 from pydantic import BaseModel
 
 from ngio.tables.backends import ImplementedTableBackends
-from ngio.utils import ZarrGroupHandler
+from ngio.tables.backends._anndata_utils import (
+    anndata_to_dataframe,
+    dataframe_to_anndata,
+)
+from ngio.utils import NgioValueError, ZarrGroupHandler
 
 
 class GenericTableMeta(BaseModel):
@@ -24,11 +29,26 @@ class GenericTable:
 
     def __init__(
         self,
-        dataframe: pd.DataFrame,
+        dataframe: pd.DataFrame | None = None,
+        anndata: AnnData | None = None,
     ) -> None:
         """Initialize the GenericTable."""
         self._meta = GenericTableMeta()
+        if dataframe is None and anndata is None:
+            raise NgioValueError(
+                "Either a DataFrame or an AnnData object must be provided."
+            )
+
+        if dataframe is not None and anndata is not None:
+            raise NgioValueError(
+                "Only one of DataFrame or AnnData object can be provided."
+            )
+
         self._dataframe = dataframe
+        self._anndata = anndata
+
+        self.anndata_native = True if anndata is not None else False
+
         self._table_backend = None
 
     @staticmethod
@@ -54,12 +74,35 @@ class GenericTable:
     @property
     def dataframe(self) -> pd.DataFrame:
         """Return the table as a DataFrame."""
-        return self._dataframe
+        if self._dataframe is not None:
+            return self._dataframe
+
+        if self._anndata is not None:
+            return anndata_to_dataframe(self._anndata)
+
+        raise NgioValueError("No table loaded.")
 
     @dataframe.setter
     def dataframe(self, dataframe: pd.DataFrame) -> None:
         """Set the table as a DataFrame."""
         self._dataframe = dataframe
+        self.anndata_native = False
+
+    @property
+    def anndata(self) -> AnnData:
+        """Return the table as an AnnData object."""
+        if self._anndata is not None:
+            return self._anndata
+
+        if self._dataframe is not None:
+            return dataframe_to_anndata(self._dataframe)
+        raise NgioValueError("No table loaded.")
+
+    @anndata.setter
+    def anndata(self, anndata: AnnData) -> None:
+        """Set the table as an AnnData object."""
+        self._anndata = anndata
+        self.anndata_native = True
 
     @classmethod
     def _from_handler(
@@ -81,12 +124,18 @@ class GenericTable:
             )
             meta.backend = backend_name
 
-        if not backend.implements_dataframe:
-            raise ValueError("The backend does not implement the dataframe protocol.")
+        if backend.implements_anndata():
+            anndata = backend.load_as_anndata()
+            table = cls(anndata=anndata)
 
-        dataframe = backend.load_as_dataframe()
+        elif backend.implements_dataframe():
+            dataframe = backend.load_as_dataframe()
+            table = cls(dataframe=dataframe)
+        else:
+            raise NgioValueError(
+                "The backend does not implement the dataframe protocol."
+            )
 
-        table = cls(dataframe)
         table._meta = meta
         table._table_backend = backend
         return table
@@ -108,11 +157,16 @@ class GenericTable:
     def consolidate(self) -> None:
         """Write the current state of the table to the Zarr file."""
         if self._table_backend is None:
-            raise ValueError(
+            raise NgioValueError(
                 "No backend set for the table. "
                 "Please add the table to a OME-Zarr Image before calling consolidate."
             )
 
-        self._table_backend.write_from_dataframe(
-            self._dataframe, metadata=self._meta.model_dump(exclude_none=True)
-        )
+        if self.anndata_native:
+            self._table_backend.write_from_anndata(
+                self.anndata, metadata=self._meta.model_dump(exclude_none=True)
+            )
+        else:
+            self._table_backend.write_from_dataframe(
+                self.dataframe, metadata=self._meta.model_dump(exclude_none=True)
+            )

ngio/tables/v1/_roi_table.py

@@ -4,8 +4,10 @@ This class follows the roi_table specification at:
 https://fractal-analytics-platform.github.io/fractal-tasks-core/tables/
 """
 
+# Import _type to avoid name conflict with table.type
+from builtins import type as _type
 from collections.abc import Iterable
-from typing import Literal
+from typing import Generic, Literal, TypeVar
 
 import pandas as pd
 from pydantic import BaseModel
@@ -89,7 +91,7 @@ def _rois_to_dataframe(rois: dict[str, WorldCooROI], index_key: str) -> pd.DataF
     return dataframe
 
 
-class ROITableV1Meta(BaseModel):
+class RoiTableV1Meta(BaseModel):
     """Metadata for the ROI table."""
 
     fractal_table_version: Literal["1"] = "1"
@@ -97,17 +99,37 @@ class ROITableV1Meta(BaseModel):
     backend: str | None = None
 
 
-class RoiTableV1:
-    """Class to handle fractal ROI tables.
+class RegionMeta(BaseModel):
+    """Metadata for the region."""
 
-    To know more about the ROI table format, please refer to the
-    specification at:
-    https://fractal-analytics-platform.github.io/fractal-tasks-core/tables/
-    """
+    path: str
 
-    def __init__(self, rois: Iterable[WorldCooROI] | None = None) -> None:
+
+class MaskingRoiTableV1Meta(BaseModel):
+    """Metadata for the ROI table."""
+
+    fractal_table_version: Literal["1"] = "1"
+    type: Literal["masking_roi_table"] = "masking_roi_table"
+    backend: str | None = None
+    region: RegionMeta | None = None
+    instance_key: str = "label"
+
+
+_roi_meta = TypeVar("_roi_meta", RoiTableV1Meta, MaskingRoiTableV1Meta)
+
+
+class _GenericRoiTableV1(Generic[_roi_meta]):
+    """Class to a non-specific table."""
+
+    _meta: _roi_meta
+
+    def __init__(
+        self, meta: _roi_meta | None = None, rois: Iterable[WorldCooROI] | None = None
+    ) -> None:
         """Create a new ROI table."""
-        self._meta = ROITableV1Meta()
+        if meta is None:
+            raise NgioValueError("Metadata must be provided.")
+        self._meta = meta
         self._table_backend = None
 
         self._rois = {}
@@ -115,15 +137,30 @@ class RoiTableV1:
             self.add(rois)
 
     @staticmethod
-    def type() -> Literal["roi_table"]:
+    def type() -> str:
         """Return the type of the table."""
-        return "roi_table"
+        raise NotImplementedError
 
     @staticmethod
     def version() -> Literal["1"]:
         """Return the version of the fractal table."""
         return "1"
 
+    @staticmethod
+    def _index_key() -> str:
+        """Return the index key of the table."""
+        raise NotImplementedError
+
+    @staticmethod
+    def _index_type() -> Literal["int", "str"]:
+        """Return the index type of the table."""
+        raise NotImplementedError
+
+    @staticmethod
+    def _meta_type() -> _type[_roi_meta]:
+        """Return the metadata type of the table."""
+        raise NotImplementedError
+
     @property
     def backend_name(self) -> str | None:
         """Return the name of the backend."""
@@ -134,23 +171,23 @@ class RoiTableV1:
     @classmethod
     def _from_handler(
         cls, handler: ZarrGroupHandler, backend_name: str | None = None
-    ) -> "RoiTableV1":
+    ) -> "_GenericRoiTableV1":
         """Create a new ROI table from a Zarr store."""
-        meta = ROITableV1Meta(**handler.load_attrs())
+        meta = cls._meta_type()(**handler.load_attrs())
 
         if backend_name is None:
             backend = ImplementedTableBackends().get_backend(
                 backend_name=meta.backend,
                 group_handler=handler,
-                index_key="FieldIndex",
-                index_type="str",
+                index_key=cls._index_key(),
+                index_type=cls._index_type(),
             )
         else:
             backend = ImplementedTableBackends().get_backend(
                 backend_name=backend_name,
                 group_handler=handler,
-                index_key="FieldIndex",
-                index_type="str",
+                index_key=cls._index_key(),
+                index_type=cls._index_type(),
             )
             meta.backend = backend_name
 
@@ -159,6 +196,7 @@ class RoiTableV1:
                 "The backend does not implement the dataframe protocol."
             )
 
+        # This will be implemented in the child classes
         table = cls()
         table._meta = meta
         table._table_backend = backend
@@ -181,8 +219,8 @@ class RoiTableV1:
         backend = ImplementedTableBackends().get_backend(
             backend_name=backend_name,
             group_handler=handler,
-            index_key="FieldIndex",
-            index_type="str",
+            index_key=self._index_key(),
+            index_type=self._index_type(),
         )
         self._meta.backend = backend_name
         self._table_backend = backend
@@ -191,12 +229,6 @@ class RoiTableV1:
         """List all ROIs in the table."""
         return list(self._rois.values())
 
-    def get(self, roi_name: str) -> WorldCooROI:
-        """Get an ROI from the table."""
-        if roi_name not in self._rois:
-            raise NgioValueError(f"ROI {roi_name} not found in the table.")
-        return self._rois[roi_name]
-
     def add(self, roi: WorldCooROI | Iterable[WorldCooROI]) -> None:
         """Append ROIs to the current table."""
         if isinstance(roi, WorldCooROI):
@@ -215,7 +247,7 @@ class RoiTableV1:
                 "Please add the table to a OME-Zarr Image before calling consolidate."
             )
 
-        dataframe = _rois_to_dataframe(self._rois, index_key="FieldIndex")
+        dataframe = _rois_to_dataframe(self._rois, index_key=self._index_key())
         dataframe = validate_columns(
             dataframe,
             required_columns=REQUIRED_COLUMNS,
@@ -224,3 +256,89 @@ class RoiTableV1:
         self._table_backend.write_from_dataframe(
             dataframe, metadata=self._meta.model_dump(exclude_none=True)
         )
+
+
+class RoiTableV1(_GenericRoiTableV1[RoiTableV1Meta]):
+    """Class to handle fractal ROI tables.
+
+    To know more about the ROI table format, please refer to the
+    specification at:
+    https://fractal-analytics-platform.github.io/fractal-tasks-core/tables/
+    """
+
+    def __init__(self, rois: Iterable[WorldCooROI] | None = None) -> None:
+        """Create a new ROI table."""
+        super().__init__(RoiTableV1Meta(), rois)
+
+    @staticmethod
+    def type() -> Literal["roi_table"]:
+        """Return the type of the table."""
+        return "roi_table"
+
+    @staticmethod
+    def _index_key() -> str:
+        """Return the index key of the table."""
+        return "FieldIndex"
+
+    @staticmethod
+    def _index_type() -> Literal["int", "str"]:
+        """Return the index type of the table."""
+        return "str"
+
+    @staticmethod
+    def _meta_type() -> _type[RoiTableV1Meta]:
+        """Return the metadata type of the table."""
+        return RoiTableV1Meta
+
+    def get(self, roi_name: str) -> WorldCooROI:
+        """Get an ROI from the table."""
+        if roi_name not in self._rois:
+            raise NgioValueError(f"ROI {roi_name} not found in the table.")
+        return self._rois[roi_name]
+
+
+class MaskingROITableV1(_GenericRoiTableV1[MaskingRoiTableV1Meta]):
+    """Class to handle fractal ROI tables.
+
+    To know more about the ROI table format, please refer to the
+    specification at:
+    https://fractal-analytics-platform.github.io/fractal-tasks-core/tables/
+    """
+
+    def __init__(
+        self,
+        rois: Iterable[WorldCooROI] | None = None,
+        reference_label: str | None = None,
+    ) -> None:
+        """Create a new ROI table."""
+        meta = MaskingRoiTableV1Meta()
+        if reference_label is not None:
+            meta.region = RegionMeta(path=reference_label)
+        super().__init__(meta, rois)
+
+    @staticmethod
+    def type() -> Literal["masking_roi_table"]:
+        """Return the type of the table."""
+        return "masking_roi_table"
+
+    @staticmethod
+    def _index_key() -> str:
+        """Return the index key of the table."""
+        return "label"
+
+    @staticmethod
+    def _index_type() -> Literal["int", "str"]:
+        """Return the index type of the table."""
+        return "int"
+
+    @staticmethod
+    def _meta_type() -> _type[MaskingRoiTableV1Meta]:
+        """Return the metadata type of the table."""
+        return MaskingRoiTableV1Meta
+
+    def get(self, label: int) -> WorldCooROI:
+        """Get an ROI from the table."""
+        _label = str(label)
+        if _label not in self._rois:
+            raise KeyError(f"ROI {_label} not found in the table.")
+        return self._rois[_label]

ngio/utils/__init__.py

@@ -11,6 +11,7 @@ from ngio.utils._errors import (
     NgioValidationError,
     NgioValueError,
 )
+from ngio.utils._fractal_fsspec_store import fractal_fsspec_store
 from ngio.utils._logger import ngio_logger, set_logger_level
 from ngio.utils._zarr_utils import (
     AccessModeLiteral,
@@ -35,6 +36,8 @@ __all__ = [
     "ZarrGroupHandler",
     # Datasets
     "download_ome_zarr_dataset",
+    # Fractal
+    "fractal_fsspec_store",
     "list_ome_zarr_datasets",
     # Logger
     "ngio_logger",

ngio/utils/_datasets.py

@@ -4,6 +4,8 @@ from pathlib import Path
 
 import pooch
 
+from ngio.utils._errors import NgioValueError
+
 _ome_zarr_zoo = {
     "CardiomyocyteTiny": {
         "url": "https://zenodo.org/records/13305156/files/20200812-CardiomyocyteDifferentiation14-Cycle1.zarr.zip",
@@ -27,7 +29,7 @@ def list_ome_zarr_datasets() -> list[str]:
 
 def download_ome_zarr_dataset(
     dataset_name: str,
-    download_dir: str = "data",
+    download_dir: str | Path = "data",
 ) -> Path:
     """Download an OME-Zarr dataset.
 
@@ -38,7 +40,7 @@ def download_ome_zarr_dataset(
         download_dir (str): The download directory. Defaults to "data".
     """
     if dataset_name not in _ome_zarr_zoo:
-        raise ValueError(f"Dataset {dataset_name} not found in the OME-Zarr zoo.")
+        raise NgioValueError(f"Dataset {dataset_name} not found in the OME-Zarr zoo.")
     ome_zarr_url = _ome_zarr_zoo[dataset_name]
     pooch.retrieve(
         path=download_dir,

ngio/utils/_fractal_fsspec_store.py

@@ -0,0 +1,13 @@
+import fsspec.implementations.http
+
+
+def fractal_fsspec_store(
+    url: str, fractal_token: str | None = None, client_kwargs: dict | None = None
+) -> fsspec.mapping.FSMap:
+    """Simple function to get an http fsspec store from a url."""
+    client_kwargs = {} if client_kwargs is None else client_kwargs
+    if fractal_token is not None:
+        client_kwargs["headers"] = {"Authorization": f"Bearer {fractal_token}"}
+    fs = fsspec.implementations.http.HTTPFileSystem(client_kwargs=client_kwargs)
+    store = fs.get_mapper(url)
+    return store

ngio/utils/_logger.py

@@ -1,5 +1,7 @@
 import logging
 
+from ngio.utils._errors import NgioValueError
+
 # Configure the logger
 ngio_logger = logging.getLogger("NgioLogger")
 ngio_logger.setLevel(logging.ERROR)
@@ -24,6 +26,6 @@ def set_logger_level(level: str) -> None:
             Must be one of "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL".
     """
     if level not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]:
-        raise ValueError(f"Invalid log level: {level}")
+        raise NgioValueError(f"Invalid log level: {level}")
 
     ngio_logger.setLevel(level)

ngio/utils/_zarr_utils.py

@@ -131,7 +131,7 @@ class ZarrGroupHandler:
                     "The store needs to be a path to use the lock mechanism."
                 )
             self._lock_path = f"{_store}.lock"
-            self._lock = FileLock(self._lock_path)
+            self._lock = FileLock(self._lock_path, timeout=10)
 
         else:
             self._lock_path = None
@@ -168,8 +168,13 @@ class ZarrGroupHandler:
         return self._mode  # type: ignore
 
     @property
-    def lock(self) -> BaseFileLock | None:
+    def lock(self) -> BaseFileLock:
         """Return the lock."""
+        if self._lock is None:
+            raise NgioValueError(
+                "The handler is not parallel safe. "
+                "Reopen the handler with parallel_safe=True."
+            )
         return self._lock
 
     @property
@@ -376,3 +381,21 @@ class ZarrGroupHandler:
             return True, self.derive_handler(path)
         except NgioError as e:
             return False, e
+
+    def copy_handler(self, handler: "ZarrGroupHandler") -> None:
+        """Copy the group to a new store."""
+        _, n_skipped, _ = zarr.copy_store(
+            source=self.group.store,
+            dest=handler.group.store,
+            source_path=self.group.path,
+            dest_path=handler.group.path,
+            if_exists="replace",
+        )
+        if n_skipped > 0:
+            raise NgioValueError(
+                f"Error copying group to {handler.full_path}, "
+                f"#{n_skipped} files where skipped."
+            )
+
+
+# %%

{ngio-0.2.0a2.dist-info → ngio-0.2.0b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngio
-Version: 0.2.0a2
+Version: 0.2.0b1
 Summary: Next Generation file format IO
 Project-URL: homepage, https://github.com/lorenzocerrone/ngio
 Project-URL: repository, https://github.com/lorenzocerrone/ngio
@@ -29,6 +29,7 @@ Requires-Dist: requests
 Requires-Dist: xarray
 Requires-Dist: zarr<3
 Provides-Extra: dev
+Requires-Dist: devtools; extra == 'dev'
 Requires-Dist: matplotlib; extra == 'dev'
 Requires-Dist: mypy; extra == 'dev'
 Requires-Dist: napari; extra == 'dev'
@@ -38,6 +39,7 @@ Requires-Dist: pre-commit; extra == 'dev'
 Requires-Dist: pyqt5; extra == 'dev'
 Requires-Dist: rich; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: scikit-image; extra == 'dev'
 Provides-Extra: docs
 Requires-Dist: mkdocs; extra == 'docs'
 Requires-Dist: mkdocs-autorefs; extra == 'docs'
@@ -50,6 +52,7 @@ Requires-Dist: scikit-image; extra == 'docs'
 Provides-Extra: test
 Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-cov; extra == 'test'
+Requires-Dist: scikit-image; extra == 'test'
 Description-Content-Type: text/markdown
 
 # NGIO - Next Generation file format IO