ngio 0.5.0__py3-none-any.whl → 0.5.0a2__py3-none-any.whl

ngio/tables/_tables_container.py

@@ -229,10 +229,10 @@ class ImplementedTables:
 
 
 class TablesContainer:
-    """A class to handle the /tables group in an OME-NGFF file."""
+    """A class to handle the /labels group in an OME-NGFF file."""
 
     def __init__(self, group_handler: ZarrGroupHandler) -> None:
-        """Initialize the TablesContainer."""
+        """Initialize the LabelGroupHandler."""
         self._group_handler = group_handler
 
         # Validate the group
@@ -252,24 +252,17 @@ class TablesContainer:
             )
 
     def _get_tables_list(self) -> list[str]:
-        """Return the list of table names from the group attributes."""
+        """Create the /tables group if it doesn't exist."""
         attrs = self._group_handler.load_attrs()
         return attrs.get("tables", [])
 
     def _get_table_group_handler(self, name: str) -> ZarrGroupHandler:
         """Get the group handler for a table."""
-        handler = self._group_handler.get_handler(path=name)
+        handler = self._group_handler.derive_handler(path=name)
         return handler
 
     def list(self, filter_types: TypedTable | str | None = None) -> list[str]:
-        """List all tables in the group.
-
-        Args:
-            filter_types: If provided, only return tables of this type.
-
-        Returns:
-            A list of table names.
-        """
+        """List all labels in the group."""
         tables = self._get_tables_list()
         if filter_types is None:
             return tables
@@ -288,16 +281,7 @@ class TablesContainer:
         backend: TableBackend | None = None,
         strict: bool = True,
     ) -> Table:
-        """Get a table from the group.
-
-        Args:
-            name: The name of the table.
-            backend: The backend to use for reading the table.
-            strict: If True, raise an error if the table type is not implemented.
-
-        Returns:
-            The table object.
-        """
+        """Get a label from the group."""
         if name not in self.list():
             raise NgioValueError(f"Table '{name}' not found in the group.")
 
@@ -317,16 +301,7 @@ class TablesContainer:
         table_cls: type[TableType],
         backend: TableBackend | None = None,
     ) -> TableType:
-        """Get a table from the group as a specific type.
-
-        Args:
-            name: The name of the table.
-            table_cls: The table class to use for loading the table.
-            backend: The backend to use for reading the table.
-
-        Returns:
-            The table object of the specified type.
-        """
+        """Get a table from the group as a specific type."""
         if name not in self.list():
             raise NgioValueError(f"Table '{name}' not found in the group.")
 
@@ -336,27 +311,6 @@ class TablesContainer:
             backend=backend,
         )  # type: ignore[return-value]
 
-    def delete(self, name: str, missing_ok: bool = False) -> None:
-        """Delete a table from the group.
-
-        Args:
-            name (str): The name of the table to delete.
-            missing_ok (bool): If True, do not raise an error if
-                the table does not exist.
-        """
-        existing_tables = self._get_tables_list()
-        if name not in existing_tables:
-            if missing_ok:
-                return
-            raise NgioValueError(
-                f"Table '{name}' not found in the Tables group. "
-                f"Available tables: {existing_tables}"
-            )
-
-        self._group_handler.delete_group(name)
-        existing_tables.remove(name)
-        self._group_handler.write_attrs({"tables": existing_tables})
-
     def add(
         self,
         name: str,
@@ -364,14 +318,7 @@ class TablesContainer:
         backend: TableBackend = DefaultTableBackend,
         overwrite: bool = False,
     ) -> None:
-        """Add a table to the group.
-
-        Args:
-            name: The name of the table.
-            table: The table object to add.
-            backend: The backend to use for writing the table.
-            overwrite: Whether to overwrite an existing table with the same name.
-        """
+        """Add a table to the group."""
         existing_tables = self._get_tables_list()
         if name in existing_tables and not overwrite:
             raise NgioValueError(
@@ -379,7 +326,9 @@ class TablesContainer:
                 "Use overwrite=True to replace it."
             )
 
-        table_handler = self._group_handler.get_handler(path=name, overwrite=overwrite)
+        table_handler = self._group_handler.derive_handler(
+            path=name, overwrite=overwrite
+        )
 
         if backend is None:
             backend = table.backend_name

ngio/tables/backends/_anndata.py

@@ -1,10 +1,8 @@
-import zarr
 from anndata import AnnData
 from anndata._settings import settings
 from pandas import DataFrame
 from polars import DataFrame as PolarsDataFrame
 from polars import LazyFrame
-from zarr.storage import FsspecStore, LocalStore, MemoryStore
 
 from ngio.tables.backends._abstract_backend import AbstractTableBackend
 from ngio.tables.backends._anndata_utils import (
@@ -15,7 +13,7 @@ from ngio.tables.backends._utils import (
     convert_polars_to_anndata,
     normalize_anndata,
 )
-from ngio.utils import NgioValueError, copy_group
+from ngio.utils import NgioValueError
 
 
 class AnnDataBackend(AbstractTableBackend):
@@ -52,66 +50,18 @@ class AnnDataBackend(AbstractTableBackend):
         """Load the table as an AnnData object."""
         return self.load_as_anndata()
 
-    def _write_to_local_store(
-        self, store: LocalStore, path: str, table: AnnData
-    ) -> None:
-        """Write the AnnData table to a LocalStore."""
-        store_path = f"{store.root}/{path}"
-        table.write_zarr(store_path)
-
-    def _write_to_fsspec_store(
-        self, store: FsspecStore, path: str, table: AnnData
-    ) -> None:
-        """Write the AnnData table to a FsspecStore."""
-        full_url = f"{store.path}/{path}"
-        fs = store.fs
-        mapper = fs.get_mapper(full_url)
-        table.write_zarr(mapper)
-
-    def _write_to_memory_store(
-        self, store: MemoryStore, path: str, table: AnnData
-    ) -> None:
-        """Write the AnnData table to a MemoryStore."""
-        store = MemoryStore()
-        table.write_zarr(store)
-        anndata_group = zarr.open_group(store, mode="r")
-        copy_group(
-            anndata_group,
-            self._group_handler._group,
-            suppress_warnings=True,
-        )
-
     def write_from_anndata(self, table: AnnData) -> None:
         """Serialize the table from an AnnData object."""
-        # Make sure to use the correct zarr format
-        settings.zarr_write_format = self._group_handler.zarr_format
-        store = self._group_handler.store
-        path = self._group_handler.group.path
-        if isinstance(store, LocalStore):
-            self._write_to_local_store(
-                store,
-                path,
-                table,
-            )
-        elif isinstance(store, FsspecStore):
-            self._write_to_fsspec_store(
-                store,
-                path,
-                table,
-            )
-        elif isinstance(store, MemoryStore):
-            self._write_to_memory_store(
-                store,
-                path,
-                table,
-            )
-        else:
+        full_url = self._group_handler.full_url
+        if full_url is None:
             raise NgioValueError(
-                f"Ngio does not support writing an AnnData table to a "
-                f"store of type {type(store)}. "
+                f"Ngio does not support writing file from a "
+                f"store of type {type(self._group_handler)}. "
                 "Please make sure to use a compatible "
-                "store like a LocalStore, or FsspecStore."
+                "store like a zarr.DirectoryStore."
             )
+        settings.zarr_write_format = self._group_handler.zarr_format
+        table.write_zarr(full_url)
 
     def write_from_pandas(self, table: DataFrame) -> None:
         """Serialize the table from a pandas DataFrame."""

ngio/tables/backends/_anndata_utils.py

@@ -9,6 +9,7 @@ from anndata._io.utils import _read_legacy_raw
 from anndata._io.zarr import read_dataframe
 from anndata.compat import _clean_uns
 from anndata.experimental import read_dispatched
+from zarr.storage import LocalStore
 
 from ngio.utils import (
     NgioValueError,
@@ -34,6 +35,10 @@ def custom_anndata_read_zarr(
         elem_to_read (Sequence[str] | None): The elements to read from the store.
     """
     group = open_group_wrapper(store=store, mode="r")
+
+    if not isinstance(group.store, LocalStore):
+        elem_to_read = ["X", "obs", "var"]
+
     if elem_to_read is None:
         elem_to_read = [
             "X",
@@ -84,7 +89,7 @@ def custom_anndata_read_zarr(
         _clean_uns(adata)
 
     if isinstance(adata, dict):
-        adata = AnnData(**adata)  # type: ignore
+        adata = AnnData(**adata)
     if not isinstance(adata, AnnData):
         raise NgioValueError(f"Expected an AnnData object, but got {type(adata)}")
     return adata

ngio/tables/backends/_csv.py

@@ -1,7 +1,20 @@
-from ngio.tables.backends._py_arrow_backends import PyArrowBackend
+import pandas as pd
+import polars as pl
 
+from ngio.tables.backends._non_zarr_backends import NonZarrBaseBackend
 
-class CsvTableBackend(PyArrowBackend):
+
+def write_lf_to_csv(path: str, table: pl.DataFrame) -> None:
+    """Write a polars DataFrame to a CSV file."""
+    table.write_csv(path)
+
+
+def write_df_to_csv(path: str, table: pd.DataFrame) -> None:
+    """Write a pandas DataFrame to a CSV file."""
+    table.to_csv(path, index=False)
+
+
+class CsvTableBackend(NonZarrBaseBackend):
     """A class to load and write small tables in CSV format."""
 
     def __init__(
@@ -9,8 +22,11 @@ class CsvTableBackend(PyArrowBackend):
     ):
         """Initialize the CsvTableBackend."""
         super().__init__(
+            lf_reader=pl.scan_csv,
+            df_reader=pd.read_csv,
+            lf_writer=write_lf_to_csv,
+            df_writer=write_df_to_csv,
             table_name="table.csv",
-            table_format="csv",
         )
 
     @staticmethod

ngio/tables/backends/_json.py

@@ -8,7 +8,7 @@ from ngio.tables.backends._utils import (
     normalize_pandas_df,
     normalize_polars_lf,
 )
-from ngio.utils import NgioError
+from ngio.utils import NgioFileNotFoundError
 
 
 class JsonTableBackend(AbstractTableBackend):
@@ -37,19 +37,22 @@ class JsonTableBackend(AbstractTableBackend):
     def _get_table_group(self):
         """Get the table group, creating it if it doesn't exist."""
         try:
-            table_group = self._group_handler.get_group(path="table", create_mode=True)
-        except NgioError as e:
-            raise NgioError(
-                "Could not get or create a 'table' group in the store "
-                f"{self._group_handler.store} path "
-                f"{self._group_handler.group.path}/table."
-            ) from e
+            table_group = self._group_handler.get_group(path="table")
+        except NgioFileNotFoundError:
+            table_group = self._group_handler.group.create_group("table")
         return table_group
 
-    def load_as_pandas_df(self) -> DataFrame:
+    def _load_as_pandas_df(self) -> DataFrame:
         """Load the table as a pandas DataFrame."""
-        table_dict = self._get_table_group().attrs.asdict()
+        table_group = self._get_table_group()
+        table_dict = dict(table_group.attrs)
+
         data_frame = pd.DataFrame.from_dict(table_dict)
+        return data_frame
+
+    def load_as_pandas_df(self) -> DataFrame:
+        """Load the table as a pandas DataFrame."""
+        data_frame = self._load_as_pandas_df()
         data_frame = normalize_pandas_df(
             data_frame,
             index_key=self.index_key,

ngio/tables/backends/_non_zarr_backends.py

@@ -0,0 +1,196 @@
+import io
+from collections.abc import Callable
+from typing import Any
+
+from pandas import DataFrame
+from polars import DataFrame as PolarsDataFrame
+from polars import LazyFrame
+from zarr.storage import FsspecStore, LocalStore
+
+from ngio.tables.backends._abstract_backend import AbstractTableBackend
+from ngio.tables.backends._utils import normalize_pandas_df, normalize_polars_lf
+from ngio.utils import NgioFileNotFoundError, NgioValueError
+
+
+class NonZarrBaseBackend(AbstractTableBackend):
+    """A class to load and write small tables in CSV format."""
+
+    def __init__(
+        self,
+        df_reader: Callable[[Any], DataFrame],
+        lf_reader: Callable[[Any], LazyFrame],
+        df_writer: Callable[[str, DataFrame], None],
+        lf_writer: Callable[[str, PolarsDataFrame], None],
+        table_name: str,
+    ):
+        self.df_reader = df_reader
+        self.lf_reader = lf_reader
+        self.df_writer = df_writer
+        self.lf_writer = lf_writer
+        self.table_name = table_name
+
+    @staticmethod
+    def implements_anndata() -> bool:
+        """Whether the handler implements the anndata protocol."""
+        return False
+
+    @staticmethod
+    def implements_pandas() -> bool:
+        """Whether the handler implements the dataframe protocol."""
+        return True
+
+    @staticmethod
+    def implements_polars() -> bool:
+        """Whether the handler implements the polars protocol."""
+        return True
+
+    @staticmethod
+    def backend_name() -> str:
+        """Return the name of the backend."""
+        raise NotImplementedError(
+            "The backend_name method must be implemented in the subclass."
+        )
+
+    def _load_from_directory_store(self, reader):
+        """Load the table from a directory store."""
+        url = self._group_handler.full_url
+        if url is None:
+            ext = self.table_name.split(".")[-1]
+            raise NgioValueError(
+                f"Ngio does not support reading a {ext} table from a "
+                f"store of type {type(self._group_handler)}. "
+                "Please make sure to use a compatible "
+                "store like a zarr.DirectoryStore."
+            )
+        table_path = f"{url}/{self.table_name}"
+        dataframe = reader(table_path)
+        return dataframe
+
+    def _load_from_fs_store_df(self, reader):
+        """Load the table from an FS store."""
+        path = self._group_handler.group.path
+        table_path = f"{path}/{self.table_name}"
+        bytes_table = self._group_handler.store.get(table_path)
+        if bytes_table is None:
+            raise NgioFileNotFoundError(f"No table found at {table_path}. ")
+        dataframe = reader(io.BytesIO(bytes_table))
+        return dataframe
+
+    def _load_from_fs_store_lf(self, reader):
+        """Load the table from an FS store."""
+        full_url = self._group_handler.full_url
+        parquet_path = f"{full_url}/{self.table_name}"
+        store_fs = self._group_handler.store.fs  # type: ignore (in this context, store_fs is a fs.FSStore)
+        with store_fs.open(parquet_path, "rb") as f:
+            dataframe = reader(f)
+        return dataframe
+
+    def load_as_pandas_df(self) -> DataFrame:
+        """Load the table as a pandas DataFrame."""
+        store = self._group_handler.store
+        if isinstance(store, LocalStore):
+            dataframe = self._load_from_directory_store(reader=self.df_reader)
+        elif isinstance(store, FsspecStore):
+            dataframe = self._load_from_fs_store_df(reader=self.df_reader)
+        else:
+            ext = self.table_name.split(".")[-1]
+            raise NgioValueError(
+                f"Ngio does not support reading a {ext} table from a "
+                f"store of type {type(store)}. "
+                "Please make sure to use a compatible "
+                "store like a zarr.DirectoryStore or "
+                "zarr.FSStore."
+            )
+
+        dataframe = normalize_pandas_df(
+            dataframe,
+            index_key=self.index_key,
+            index_type=self.index_type,
+            reset_index=False,
+        )
+        return dataframe
+
+    def load(self) -> DataFrame:
+        """Load the table as a pandas DataFrame."""
+        return self.load_as_pandas_df()
+
+    def load_as_polars_lf(self) -> LazyFrame:
+        """Load the table as a polars LazyFrame."""
+        store = self._group_handler.store
+        if isinstance(store, LocalStore):
+            lazy_frame = self._load_from_directory_store(reader=self.lf_reader)
+        elif isinstance(store, FsspecStore):
+            lazy_frame = self._load_from_fs_store_lf(reader=self.lf_reader)
+        else:
+            ext = self.table_name.split(".")[-1]
+            raise NgioValueError(
+                f"Ngio does not support reading a {ext} from a "
+                f"store of type {type(store)}. "
+                "Please make sure to use a compatible "
+                "store like a zarr.DirectoryStore or "
+                "zarr.FSStore."
+            )
+        if not isinstance(lazy_frame, LazyFrame):
+            raise NgioValueError(
+                "Table is not a lazy frame. Please report this issue as an ngio bug."
+                f" {type(lazy_frame)}"
+            )
+
+        lazy_frame = normalize_polars_lf(
+            lazy_frame,
+            index_key=self.index_key,
+            index_type=self.index_type,
+        )
+        return lazy_frame
+
+    def _get_store_url(self) -> str:
+        """Get the store URL."""
+        store = self._group_handler.store
+        if isinstance(store, LocalStore):
+            full_url = self._group_handler.full_url
+        else:
+            ext = self.table_name.split(".")[-1]
+            raise NgioValueError(
+                f"Ngio does not support writing a {ext} file to a "
+                f"store of type {type(store)}. "
+                "Please make sure to use a compatible "
+                "store like a zarr.DirectoryStore or "
+                "zarr.FSStore."
+            )
+        if full_url is None:
+            ext = self.table_name.split(".")[-1]
+            raise NgioValueError(
+                f"Ngio does not support writing a {ext} file to a "
+                f"store of type {type(store)}. "
+                "Please make sure to use a compatible "
+                "store like a zarr.DirectoryStore or "
+                "zarr.FSStore."
+            )
+        return full_url
+
+    def write_from_pandas(self, table: DataFrame) -> None:
+        """Write the table from a pandas DataFrame."""
+        table = normalize_pandas_df(
+            table,
+            index_key=self.index_key,
+            index_type=self.index_type,
+            reset_index=True,
+        )
+        full_url = self._get_store_url()
+        table_path = f"{full_url}/{self.table_name}"
+        self.df_writer(table_path, table)
+
+    def write_from_polars(self, table: PolarsDataFrame | LazyFrame) -> None:
+        """Write the table from a polars DataFrame or LazyFrame."""
+        table = normalize_polars_lf(
+            table,
+            index_key=self.index_key,
+            index_type=self.index_type,
+        )
+
+        if isinstance(table, LazyFrame):
+            table = table.collect()
+
+        full_url = self._get_store_url()
+        table_path = f"{full_url}/{self.table_name}"
+        self.lf_writer(table_path, table)

ngio/tables/backends/_parquet.py

@@ -1,7 +1,32 @@
-from ngio.tables.backends._py_arrow_backends import PyArrowBackend
+import pandas as pd
+import polars as pl
 
+from ngio.tables.backends._non_zarr_backends import NonZarrBaseBackend
 
-class ParquetTableBackend(PyArrowBackend):
+
+def write_lf_to_parquet(path: str, table: pl.DataFrame) -> None:
+    """Write a polars DataFrame to a Parquet file."""
+    # make categorical into string (for pandas compatibility)
+    schema = table.collect_schema()
+
+    categorical_columns = []
+    for name, dtype in zip(schema.names(), schema.dtypes(), strict=True):
+        if dtype == pl.Categorical:
+            categorical_columns.append(name)
+
+    for col in categorical_columns:
+        table = table.with_columns(pl.col(col).cast(pl.Utf8))
+
+    # write to parquet
+    table.write_parquet(path)
+
+
+def write_df_to_parquet(path: str, table: pd.DataFrame) -> None:
+    """Write a pandas DataFrame to a Parquet file."""
+    table.to_parquet(path, index=False)
+
+
+class ParquetTableBackend(NonZarrBaseBackend):
     """A class to load and write small tables in Parquet format."""
 
     def __init__(
@@ -9,8 +34,11 @@ class ParquetTableBackend(PyArrowBackend):
     ):
         """Initialize the ParquetTableBackend."""
         super().__init__(
+            lf_reader=pl.scan_parquet,
+            df_reader=pd.read_parquet,
+            lf_writer=write_lf_to_parquet,
+            df_writer=write_df_to_parquet,
             table_name="table.parquet",
-            table_format="parquet",
         )
 
     @staticmethod

ngio/tables/v1/_roi_table.py

@@ -4,7 +4,6 @@ This class follows the roi_table specification at:
 https://fractal-analytics-platform.github.io/fractal-tasks-core/tables/
 """
 
-import warnings
 from collections.abc import Iterable
 from typing import Literal
 from uuid import uuid4
@@ -27,6 +26,7 @@ from ngio.utils import (
     NgioTableValidationError,
     NgioValueError,
     ZarrGroupHandler,
+    ngio_warn,
 )
 
 REQUIRED_COLUMNS = [
@@ -77,9 +77,7 @@ OPTIONAL_COLUMNS = ORIGIN_COLUMNS + TRANSLATION_COLUMNS + PLATE_COLUMNS + INDEX_
 def _check_optional_columns(col_name: str) -> None:
     """Check if the column name is in the optional columns."""
     if col_name not in OPTIONAL_COLUMNS + TIME_COLUMNS:
-        warnings.warn(
-            f"Column {col_name} is not in the optional columns.", stacklevel=2
-        )
+        ngio_warn(f"Column {col_name} is not in the optional columns.")
 
 
 def _dataframe_to_rois(
@@ -122,17 +120,17 @@ def _dataframe_to_rois(
         else:
             label = getattr(row, "label", None)
 
-        slices = {
-            "x": (row.x_micrometer, row.len_x_micrometer),
-            "y": (row.y_micrometer, row.len_y_micrometer),
-            "z": (z_micrometer, z_length_micrometer),
-        }
-        if t_second is not None or t_length_second is not None:
-            slices["t"] = (t_second, t_length_second)
-        roi = Roi.from_values(
+        roi = Roi(
             name=str(row.Index),
-            slices=slices,
-            space="world",
+            x=row.x_micrometer,  # type: ignore (type can not be known here)
+            y=row.y_micrometer,  # type: ignore (type can not be known here)
+            z=z_micrometer,
+            t=t_second,
+            x_length=row.len_x_micrometer,  # type: ignore (type can not be known here)
+            y_length=row.len_y_micrometer,  # type: ignore (type can not be known here)
+            z_length=z_length_micrometer,
+            t_length=t_length_second,
+            unit="micrometer",
             label=label,
             **extras,
         )
@@ -145,39 +143,24 @@ def _rois_to_dataframe(rois: dict[str, Roi], index_key: str | None) -> pd.DataFr
     data = []
     for roi in rois.values():
         # This normalization is necessary for backward compatibility
-        if roi.space != "world":
-            raise NotImplementedError(
-                "Only ROIs in world coordinates can be serialized."
-            )
+        z_micrometer = roi.z if roi.z is not None else 0.0
+        len_z_micrometer = roi.z_length if roi.z_length is not None else 1.0
 
-        z_slice = roi.get("z")
-        if z_slice is None:
-            z_micrometer = 0.0
-            len_z_micrometer = 1.0
-        else:
-            z_micrometer = z_slice.start if z_slice.start is not None else 0.0
-            len_z_micrometer = z_slice.length if z_slice.length is not None else 1.0
-
-        x_slice = roi.get("x")
-        if x_slice is None:
-            raise NgioValueError("ROI is missing 'x' slice.")
-        y_slice = roi.get("y")
-        if y_slice is None:
-            raise NgioValueError("ROI is missing 'y' slice.")
         row = {
             index_key: roi.get_name(),
-            "x_micrometer": x_slice.start if x_slice.start is not None else 0.0,
-            "y_micrometer": y_slice.start if y_slice.start is not None else 0.0,
+            "x_micrometer": roi.x,
+            "y_micrometer": roi.y,
             "z_micrometer": z_micrometer,
-            "len_x_micrometer": x_slice.length if x_slice.length is not None else 1.0,
-            "len_y_micrometer": y_slice.length if y_slice.length is not None else 1.0,
+            "len_x_micrometer": roi.x_length,
+            "len_y_micrometer": roi.y_length,
             "len_z_micrometer": len_z_micrometer,
         }
 
-        t_slice = roi.get("t")
-        if t_slice is not None:
-            row["t_second"] = t_slice.start if t_slice.start is not None else 0.0
-            row["len_t_second"] = t_slice.length if t_slice.length is not None else 1.0
+        if roi.t is not None:
+            row["t_second"] = roi.t
+
+        if roi.t_length is not None:
+            row["len_t_second"] = roi.t_length
 
         if roi.label is not None and index_key != "label":
             row["label"] = roi.label
@@ -200,7 +183,7 @@ class RoiDictWrapper:
         self._rois_by_name = {}
         self._rois_by_label = {}
         for roi in rois:
-            name = roi.name
+            name = roi.get_name()
             if name in self._rois_by_name:
                 name = f"{name}_{uuid4().hex[:8]}"
             self._rois_by_name[name] = roi