ngio 0.2.9__py3-none-any.whl → 0.3.0a1__py3-none-any.whl

ngio/tables/v1/_roi_table.py

@@ -5,17 +5,30 @@ https://fractal-analytics-platform.github.io/fractal-tasks-core/tables/
 """
 
 # Import _type to avoid name conflict with table.type
-from builtins import type as _type
 from collections.abc import Iterable
-from typing import Generic, Literal, TypeVar
+from functools import cache
+from typing import Literal
 
 import pandas as pd
 from pydantic import BaseModel
 
 from ngio.common import Roi
-from ngio.tables._validators import validate_columns
-from ngio.tables.backends import BackendMeta, ImplementedTableBackends
-from ngio.utils import NgioValueError, ZarrGroupHandler
+from ngio.tables.abstract_table import (
+    AbstractBaseTable,
+    TabularData,
+)
+from ngio.tables.backends import (
+    BackendMeta,
+    TableBackend,
+    convert_to_pandas,
+    normalize_pandas_df,
+)
+from ngio.utils import (
+    NgioTableValidationError,
+    NgioValueError,
+    ZarrGroupHandler,
+    ngio_logger,
+)
 
 REQUIRED_COLUMNS = [
     "x_micrometer",
@@ -26,6 +39,10 @@ REQUIRED_COLUMNS = [
     "len_z_micrometer",
 ]
 
+#####################
+# Optional columns are not validated at the moment
+# only a warning is raised if non optional columns are present
+#####################
 
 ORIGIN_COLUMNS = [
     "x_micrometer_original",
@@ -34,36 +51,95 @@ ORIGIN_COLUMNS = [
 
 TRANSLATION_COLUMNS = ["translation_x", "translation_y", "translation_z"]
 
-OPTIONAL_COLUMNS = ORIGIN_COLUMNS + TRANSLATION_COLUMNS
+PLATE_COLUMNS = ["plate_name", "row", "column", "path", "acquisition"]
 
+INDEX_COLUMNS = [
+    "FieldIndex",
+    "label",
+]
+
+OPTIONAL_COLUMNS = ORIGIN_COLUMNS + TRANSLATION_COLUMNS + PLATE_COLUMNS + INDEX_COLUMNS
+
+
+@cache
+def _check_optional_columns(col_name: str) -> None:
+    """Check if the column name is in the optional columns."""
+    if col_name not in OPTIONAL_COLUMNS:
+        ngio_logger.warning(
+            f"Column {col_name} is not in the optional columns. "
+            f"Standard optional columns are: {OPTIONAL_COLUMNS}."
+        )
 
-def _dataframe_to_rois(dataframe: pd.DataFrame) -> dict[str, Roi]:
+
+def _dataframe_to_rois(
+    dataframe: pd.DataFrame,
+    required_columns: list[str] | None = None,
+) -> dict[str, Roi]:
     """Convert a DataFrame to a WorldCooROI object."""
+    if required_columns is None:
+        required_columns = REQUIRED_COLUMNS
+
+    # Validate the columns of the DataFrame
+    _required_columns = set(dataframe.columns).intersection(set(required_columns))
+    if len(_required_columns) != len(required_columns):
+        raise NgioTableValidationError(
+            f"Could not find required columns: {_required_columns} in the table."
+        )
+
+    extra_columns = set(dataframe.columns).difference(set(required_columns))
+
+    for col in extra_columns:
+        _check_optional_columns(col)
+
+    extras = {}
+
     rois = {}
-    for key, row in dataframe.iterrows():
+    for row in dataframe.itertuples(index=True):
         # check if optional columns are present
-        origin = {col: row.get(col, None) for col in ORIGIN_COLUMNS}
-        origin = dict(filter(lambda x: x[1] is not None, origin.items()))
-        translation = {col: row.get(col, None) for col in TRANSLATION_COLUMNS}
-        translation = dict(filter(lambda x: x[1] is not None, translation.items()))
+        if len(extra_columns) > 0:
+            extras = {col: getattr(row, col, None) for col in extra_columns}
 
         roi = Roi(
-            name=str(key),
-            x=row["x_micrometer"],
-            y=row["y_micrometer"],
-            z=row["z_micrometer"],
-            x_length=row["len_x_micrometer"],
-            y_length=row["len_y_micrometer"],
-            z_length=row["len_z_micrometer"],
+            name=str(row.Index),
+            x=row.x_micrometer,  # type: ignore
+            y=row.y_micrometer,  # type: ignore
+            z=row.z_micrometer,  # type: ignore
+            x_length=row.len_x_micrometer,  # type: ignore
+            y_length=row.len_y_micrometer,  # type: ignore
+            z_length=row.len_z_micrometer,  # type: ignore
             unit="micrometer",  # type: ignore
-            **origin,
-            **translation,
+            **extras,
         )
         rois[roi.name] = roi
     return rois
 
 
-def _rois_to_dataframe(rois: dict[str, Roi], index_key: str) -> pd.DataFrame:
+def _table_to_rois(
+    table: TabularData,
+    index_key: str | None = None,
+    index_type: Literal["int", "str"] | None = None,
+    required_columns: list[str] | None = None,
+) -> tuple[pd.DataFrame, dict[str, Roi]]:
+    """Convert a table to a dictionary of ROIs.
+
+    Args:
+        table: The table to convert.
+        index_key: The column name to use as the index of the DataFrame.
+        index_type: The type of the index column in the DataFrame.
+        required_columns: The required columns in the DataFrame.
+
+    Returns:
+        A dictionary of ROIs.
+    """
+    dataframe = convert_to_pandas(
+        table,
+        index_key=index_key,
+        index_type=index_type,
+    )
+    return dataframe, _dataframe_to_rois(dataframe, required_columns=required_columns)
+
+
+def _rois_to_dataframe(rois: dict[str, Roi], index_key: str | None) -> pd.DataFrame:
     """Convert a list of WorldCooROI objects to a DataFrame."""
     data = []
     for roi in rois.values():
@@ -78,153 +154,110 @@ def _rois_to_dataframe(rois: dict[str, Roi], index_key: str) -> pd.DataFrame:
         }
 
         extra = roi.model_extra or {}
-        for col in ORIGIN_COLUMNS:
-            if col in extra:
-                row[col] = extra[col]
-
-        for col in TRANSLATION_COLUMNS:
-            if col in extra:
-                row[col] = extra[col]
+        for col in extra:
+            _check_optional_columns(col)
+            row[col] = extra[col]
         data.append(row)
     dataframe = pd.DataFrame(data)
-    dataframe = dataframe.set_index(index_key)
+    dataframe = normalize_pandas_df(dataframe, index_key=index_key)
     return dataframe
 
 
-class RoiTableV1Meta(BackendMeta):
-    """Metadata for the ROI table."""
-
-    fractal_table_version: Literal["1"] = "1"
-    type: Literal["roi_table"] = "roi_table"
-
-
-class RegionMeta(BaseModel):
-    """Metadata for the region."""
-
-    path: str
-
-
-class MaskingRoiTableV1Meta(BackendMeta):
-    """Metadata for the ROI table."""
-
-    fractal_table_version: Literal["1"] = "1"
-    type: Literal["masking_roi_table"] = "masking_roi_table"
-    region: RegionMeta | None = None
-    instance_key: str = "label"
-
-
-_roi_meta = TypeVar("_roi_meta", RoiTableV1Meta, MaskingRoiTableV1Meta)
-
-
-class _GenericRoiTableV1(Generic[_roi_meta]):
-    """Class to a non-specific table."""
-
-    _meta: _roi_meta
-
+class GenericRoiTableV1(AbstractBaseTable):
     def __init__(
-        self, meta: _roi_meta | None = None, rois: Iterable[Roi] | None = None
+        self,
+        *,
+        rois: Iterable[Roi] | None = None,
+        meta: BackendMeta,
     ) -> None:
-        """Create a new ROI table."""
-        if meta is None:
-            raise NgioValueError("Metadata must be provided.")
-        self._meta = meta
-        self._table_backend = None
+        table = None
 
-        self._rois = {}
+        self._rois: dict[str, Roi] | None = None
         if rois is not None:
+            self._rois = {}
             self.add(rois)
+            table = _rois_to_dataframe(self._rois, index_key=meta.index_key)
+
+        super().__init__(table_data=table, meta=meta)
+
+    def __repr__(self) -> str:
+        """Return a string representation of the table."""
+        rois = self.rois()
+        prop = f"num_rois={len(rois)}"
+        class_name = self.__class__.__name__
+        return f"{class_name}({prop})"
 
     @staticmethod
-    def type() -> str:
+    def table_type() -> str:
         """Return the type of the table."""
-        raise NotImplementedError
+        return "generic_roi_table"
 
     @staticmethod
     def version() -> Literal["1"]:
         """Return the version of the fractal table."""
         return "1"
 
-    @staticmethod
-    def _index_key() -> str:
-        """Return the index key of the table."""
-        raise NotImplementedError
-
-    @staticmethod
-    def _index_type() -> Literal["int", "str"]:
-        """Return the index type of the table."""
-        raise NotImplementedError
+    @property
+    def table_data(self) -> TabularData:
+        """Return the table."""
+        if self._rois is None:
+            return super().table_data
 
-    @staticmethod
-    def _meta_type() -> _type[_roi_meta]:
-        """Return the metadata type of the table."""
-        raise NotImplementedError
+        if len(self.rois()) > 0:
+            self._table_data = _rois_to_dataframe(self._rois, index_key=self.index_key)
+        return super().table_data
 
-    @property
-    def backend_name(self) -> str | None:
-        """Return the name of the backend."""
-        if self._table_backend is None:
+    def set_table_data(
+        self, table_data: TabularData | None = None, refresh: bool = False
+    ) -> None:
+        if table_data is not None:
+            if not isinstance(table_data, TabularData):
+                raise NgioValueError(
+                    "The table must be a pandas DataFrame, polars LazyFrame, "
+                    " or AnnData object."
+                )
+
+            table_data, rois = _table_to_rois(
+                table_data,
+                index_key=self.index_key,
+                index_type=self.index_type,
+                required_columns=REQUIRED_COLUMNS,
+            )
+            self._table_data = table_data
+            self._rois = rois
             return None
-        return self._table_backend.backend_name()
 
-    @classmethod
-    def _from_handler(
-        cls, handler: ZarrGroupHandler, backend_name: str | None = None
-    ) -> "_GenericRoiTableV1":
-        """Create a new ROI table from a Zarr store."""
-        meta = cls._meta_type()(**handler.load_attrs())
-
-        if backend_name is None:
-            backend = ImplementedTableBackends().get_backend(
-                backend_name=meta.backend,
-                group_handler=handler,
-                index_key=cls._index_key(),
-                index_type=cls._index_type(),
-            )
-        else:
-            backend = ImplementedTableBackends().get_backend(
-                backend_name=backend_name,
-                group_handler=handler,
-                index_key=cls._index_key(),
-                index_type=cls._index_type(),
-            )
-            meta.backend = backend_name
+        if self._table_data is not None and not refresh:
+            return None
 
-        if not backend.implements_pandas:
+        if self._table_backend is None:
             raise NgioValueError(
-                "The backend does not implement the dataframe protocol."
+                "The table does not have a DataFrame in memory nor a backend."
             )
 
-        # This will be implemented in the child classes
-        table = cls()
-        table._meta = meta
-        table._table_backend = backend
-
-        dataframe = backend.load_as_pandas_df()
-        dataframe = validate_columns(
-            dataframe,
+        table_data, rois = _table_to_rois(
+            self._table_backend.load(),
+            index_key=self.index_key,
+            index_type=self.index_type,
             required_columns=REQUIRED_COLUMNS,
-            optional_columns=OPTIONAL_COLUMNS,
         )
-        table._rois = _dataframe_to_rois(dataframe)
-        return table
+        self._table_data = table_data
+        self._rois = rois
 
-    def _set_backend(
-        self,
-        handler: ZarrGroupHandler,
-        backend_name: str | None = None,
-    ) -> None:
-        """Set the backend of the table."""
-        backend = ImplementedTableBackends().get_backend(
-            backend_name=backend_name,
-            group_handler=handler,
-            index_key=self._index_key(),
-            index_type=self._index_type(),
-        )
-        self._meta.backend = backend_name
-        self._table_backend = backend
+    def _check_rois(self) -> None:
+        """Load the ROIs from the table.
+
+        If the ROIs are already loaded, do nothing.
+        If the ROIs are not loaded, load them from the table.
+        """
+        if self._rois is None:
+            self._rois = _dataframe_to_rois(self.dataframe)
 
     def rois(self) -> list[Roi]:
         """List all ROIs in the table."""
+        self._check_rois()
+        if self._rois is None:
+            return []
         return list(self._rois.values())
 
     def add(self, roi: Roi | Iterable[Roi], overwrite: bool = False) -> None:
@@ -237,31 +270,49 @@ class _GenericRoiTableV1(Generic[_roi_meta]):
         if isinstance(roi, Roi):
             roi = [roi]
 
+        self._check_rois()
+        if self._rois is None:
+            self._rois = {}
+
         for _roi in roi:
             if not overwrite and _roi.name in self._rois:
                 raise NgioValueError(f"ROI {_roi.name} already exists in the table.")
             self._rois[_roi.name] = _roi
 
-    def consolidate(self) -> None:
-        """Write the current state of the table to the Zarr file."""
-        if self._table_backend is None:
-            raise NgioValueError(
-                "No backend set for the table. "
-                "Please add the table to a OME-Zarr Image before calling consolidate."
-            )
+    def get(self, roi_name: str) -> Roi:
+        """Get an ROI from the table."""
+        self._check_rois()
+        if self._rois is None:
+            self._rois = {}
 
-        dataframe = _rois_to_dataframe(self._rois, index_key=self._index_key())
-        dataframe = validate_columns(
-            dataframe,
+        if roi_name not in self._rois:
+            raise NgioValueError(f"ROI {roi_name} not found in the table.")
+        return self._rois[roi_name]
+
+    @classmethod
+    def from_table_data(
+        cls, table_data: TabularData, meta: BackendMeta
+    ) -> "GenericRoiTableV1":
+        """Create a new ROI table from a table data."""
+        _, rois = _table_to_rois(
+            table=table_data,
+            index_key=meta.index_key,
+            index_type=meta.index_type,
             required_columns=REQUIRED_COLUMNS,
-            optional_columns=OPTIONAL_COLUMNS,
-        )
-        self._table_backend.write(
-            dataframe, metadata=self._meta.model_dump(exclude_none=True), mode="pandas"
         )
+        return cls(rois=rois.values(), meta=meta)
+
 
+class RoiTableV1Meta(BackendMeta):
+    """Metadata for the ROI table."""
 
-class RoiTableV1(_GenericRoiTableV1[RoiTableV1Meta]):
+    table_version: Literal["1"] = "1"
+    type: Literal["roi_table"] = "roi_table"
+    index_key: str | None = "FieldIndex"
+    index_type: Literal["str", "int"] | None = "str"
+
+
+class RoiTableV1(GenericRoiTableV1):
     """Class to handle fractal ROI tables.
 
     To know more about the ROI table format, please refer to the
@@ -269,43 +320,57 @@ class RoiTableV1(_GenericRoiTableV1[RoiTableV1Meta]):
     https://fractal-analytics-platform.github.io/fractal-tasks-core/tables/
     """
 
-    def __init__(self, rois: Iterable[Roi] | None = None) -> None:
+    def __init__(
+        self, rois: Iterable[Roi] | None = None, *, meta: RoiTableV1Meta | None = None
+    ) -> None:
         """Create a new ROI table."""
-        super().__init__(RoiTableV1Meta(), rois)
+        if meta is None:
+            meta = RoiTableV1Meta()
 
-    def __repr__(self) -> str:
-        """Return a string representation of the table."""
-        prop = f"num_rois={len(self._rois)}"
-        return f"RoiTableV1({prop})"
+        if meta.index_key is None:
+            meta.index_key = "FieldIndex"
+
+        if meta.index_type is None:
+            meta.index_type = "str"
+        super().__init__(meta=meta, rois=rois)
+
+    @classmethod
+    def from_handler(
+        cls,
+        handler: ZarrGroupHandler,
+        backend: TableBackend | None = None,
+    ) -> "RoiTableV1":
+        table = cls._from_handler(
+            handler=handler,
+            backend=backend,
+            meta_model=RoiTableV1Meta,
+        )
+        return table
 
     @staticmethod
-    def type() -> Literal["roi_table"]:
+    def table_type() -> Literal["roi_table"]:
         """Return the type of the table."""
         return "roi_table"
 
-    @staticmethod
-    def _index_key() -> str:
-        """Return the index key of the table."""
-        return "FieldIndex"
 
-    @staticmethod
-    def _index_type() -> Literal["int", "str"]:
-        """Return the index type of the table."""
-        return "str"
+class RegionMeta(BaseModel):
+    """Metadata for the region."""
 
-    @staticmethod
-    def _meta_type() -> _type[RoiTableV1Meta]:
-        """Return the metadata type of the table."""
-        return RoiTableV1Meta
+    path: str
 
-    def get(self, roi_name: str) -> Roi:
-        """Get an ROI from the table."""
-        if roi_name not in self._rois:
-            raise NgioValueError(f"ROI {roi_name} not found in the table.")
-        return self._rois[roi_name]
+
+class MaskingRoiTableV1Meta(BackendMeta):
+    """Metadata for the ROI table."""
+
+    table_version: Literal["1"] = "1"
+    type: Literal["masking_roi_table"] = "masking_roi_table"
+    region: RegionMeta | None = None
+    instance_key: str = "label"
+    index_key: str | None = "label"
+    index_type: Literal["int", "str"] | None = "int"
 
 
-class MaskingRoiTableV1(_GenericRoiTableV1[MaskingRoiTableV1Meta]):
+class MaskingRoiTableV1(GenericRoiTableV1):
     """Class to handle fractal ROI tables.
 
     To know more about the ROI table format, please refer to the
@@ -316,54 +381,73 @@ class MaskingRoiTableV1(_GenericRoiTableV1[MaskingRoiTableV1Meta]):
     def __init__(
         self,
         rois: Iterable[Roi] | None = None,
+        *,
         reference_label: str | None = None,
+        meta: MaskingRoiTableV1Meta | None = None,
     ) -> None:
         """Create a new ROI table."""
-        meta = MaskingRoiTableV1Meta()
+        if meta is None:
+            meta = MaskingRoiTableV1Meta()
+
         if reference_label is not None:
             meta.region = RegionMeta(path=reference_label)
-        super().__init__(meta, rois)
+
+        if meta.index_key is None:
+            meta.index_key = "label"
+
+        if meta.index_type is None:
+            meta.index_type = "int"
+        meta.instance_key = meta.index_key
+        super().__init__(meta=meta, rois=rois)
 
     def __repr__(self) -> str:
         """Return a string representation of the table."""
-        prop = f"num_rois={len(self._rois)}"
+        rois = self.rois()
         if self.reference_label is not None:
-            prop += f", reference_label={self.reference_label}"
+            prop = f"num_rois={len(rois)}, reference_label={self.reference_label}"
+        else:
+            prop = f"num_rois={len(rois)}"
         return f"MaskingRoiTableV1({prop})"
 
+    @classmethod
+    def from_handler(
+        cls,
+        handler: ZarrGroupHandler,
+        backend: TableBackend | None = None,
+    ) -> "MaskingRoiTableV1":
+        table = cls._from_handler(
+            handler=handler,
+            backend=backend,
+            meta_model=MaskingRoiTableV1Meta,
+        )
+        return table
+
     @staticmethod
-    def type() -> Literal["masking_roi_table"]:
+    def table_type() -> Literal["masking_roi_table"]:
         """Return the type of the table."""
         return "masking_roi_table"
 
-    @staticmethod
-    def _index_key() -> str:
-        """Return the index key of the table."""
-        return "label"
-
-    @staticmethod
-    def _index_type() -> Literal["int", "str"]:
-        """Return the index type of the table."""
-        return "int"
-
-    @staticmethod
-    def _meta_type() -> _type[MaskingRoiTableV1Meta]:
-        """Return the metadata type of the table."""
-        return MaskingRoiTableV1Meta
+    @property
+    def meta(self) -> MaskingRoiTableV1Meta:
+        """Return the metadata of the table."""
+        if not isinstance(self._meta, MaskingRoiTableV1Meta):
+            raise NgioValueError(
+                "The metadata of the table is not of type MaskingRoiTableV1Meta."
+            )
+        return self._meta
 
     @property
     def reference_label(self) -> str | None:
         """Return the reference label."""
-        path = self._meta.region
+        path = self.meta.region
         if path is None:
             return None
+
         path = path.path
         path = path.split("/")[-1]
         return path
 
-    def get(self, label: int) -> Roi:
+    def get(self, label: int | str) -> Roi:  # type: ignore
         """Get an ROI from the table."""
-        _label = str(label)
-        if _label not in self._rois:
-            raise KeyError(f"ROI {_label} not found in the table.")
-        return self._rois[_label]
+        roi_name = str(label)
+        return super().get(roi_name)

ngio/utils/_fractal_fsspec_store.py

@@ -1,4 +1,7 @@
 import fsspec.implementations.http
+from aiohttp import ClientResponseError
+
+from ngio.utils import NgioValueError
 
 
 def fractal_fsspec_store(
@@ -9,5 +12,31 @@ def fractal_fsspec_store(
     if fractal_token is not None:
         client_kwargs["headers"] = {"Authorization": f"Bearer {fractal_token}"}
     fs = fsspec.implementations.http.HTTPFileSystem(client_kwargs=client_kwargs)
+
     store = fs.get_mapper(url)
+
+    possible_keys = [".zgroup", ".zarray"]
+    for key in possible_keys:
+        try:
+            value = store.get(key)
+            if value is not None:
+                break
+        except ClientResponseError as e:
+            if e.status == 401 and fractal_token is None:
+                raise NgioValueError(
+                    "No auto token is provided. You need a valid "
+                    f"'fractal_token' to access: {url}."
+                ) from e
+            elif e.status == 401 and fractal_token is not None:
+                raise NgioValueError(
+                    f"The 'fractal_token' provided is invalid for: {url}."
+                ) from e
+            else:
+                raise e
+    else:
+        raise NgioValueError(
+            f"Store {url} can not be read. Possible problems are: \n"
+            "- The url does not exist. \n"
+            f"- The url is not a valid .zarr. \n"
+        )
     return store