ngio 0.1.6__py3-none-any.whl → 0.2.0a1__py3-none-any.whl

ngio/tables/_ad_reader.py

@@ -1,80 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any
-
-import zarr
-from anndata import AnnData
-from anndata._io.specs import read_elem
-from anndata._io.utils import _read_legacy_raw
-from anndata._io.zarr import read_dataframe
-from anndata.compat import _clean_uns
-from anndata.experimental import read_dispatched
-
-from ngio.io import open_group_wrapper
-
-if TYPE_CHECKING:
-    from collections.abc import Callable
-
-    from ngio.io import StoreOrGroup
-
-
-def custom_read_zarr(store: StoreOrGroup) -> AnnData:
-    """Read from a hierarchical Zarr array store.
-
-    # Implementation originally from https://github.com/scverse/anndata/blob/main/src/anndata/_io/zarr.py
-    # Original implementation would not work with remote storages so we had to copy it
-    # here and slightly modified it to work with remote storages.
-
-    Args:
-        store (StoreOrGroup): A store or group to read the AnnData from.
-    """
-    group = open_group_wrapper(store=store, mode="r")
-
-    # Read with handling for backwards compat
-    def callback(func: Callable, elem_name: str, elem: Any, iospec: Any) -> Any:
-        if iospec.encoding_type == "anndata" or elem_name.endswith("/"):
-            ad_kwargs = {}
-            # Some of these elem fail on https
-            # So we only include the ones that are strictly necessary
-            # for fractal tables
-            base_elem = [
-                "X",
-                # "layers",
-                "obs",
-                # "obsm",
-                # "obsp",
-                # "uns",
-                "var",
-                # "varm",
-                # "varp",
-            ]
-            # This fails on some https
-            # base_elem += list(elem.keys())
-            for k in set(base_elem):
-                v = elem.get(k)
-                if v is not None and not k.startswith("raw."):
-                    ad_kwargs[k] = read_dispatched(v, callback)
-            return AnnData(**ad_kwargs)
-
-        elif elem_name.startswith("/raw."):
-            return None
-        elif elem_name in {"/obs", "/var"}:
-            return read_dataframe(elem)
-        elif elem_name == "/raw":
-            # Backwards compat
-            return _read_legacy_raw(group, func(elem), read_dataframe, func)
-        return func(elem)
-
-    adata = read_dispatched(group, callback=callback)
-
-    # Backwards compat (should figure out which version)
-    if "raw.X" in group:
-        raw = AnnData(**_read_legacy_raw(group, adata.raw, read_dataframe, read_elem))
-        raw.obs_names = adata.obs_names
-        adata.raw = raw
-
-    # Backwards compat for <0.7
-    if isinstance(group["obs"], zarr.Array):
-        _clean_uns(adata)
-
-    return adata

ngio/tables/_utils.py

@@ -1,301 +0,0 @@
-from collections.abc import Callable
-from typing import Literal
-
-import anndata as ad
-import numpy as np
-import pandas as pd
-import pandas.api.types as ptypes
-
-from ngio.utils import NgioTableValidationError
-
-Validator = Callable[[pd.DataFrame], pd.DataFrame]
-
-
-def _check_for_mixed_types(series: pd.Series) -> None:
-    """Check if the column has mixed types."""
-    if series.apply(type).nunique() > 1:
-        raise NgioTableValidationError(
-            f"Column {series.name} has mixed types: "
-            f"{series.apply(type).unique()}. "
-            "Type of all elements must be the same."
-        )
-
-
-def _check_for_supported_types(series: pd.Series) -> Literal["str", "int", "numeric"]:
-    """Check if the column has supported types."""
-    if ptypes.is_string_dtype(series):
-        return "str"
-    if ptypes.is_integer_dtype(series):
-        return "int"
-    if ptypes.is_numeric_dtype(series):
-        return "numeric"
-    raise NgioTableValidationError(
-        f"Column {series.name} has unsupported type: {series.dtype}."
-        " Supported types are string and numerics."
-    )
-
-
-def _check_index_key(
-    table_df: pd.DataFrame, index_key: str, index_type: Literal["str", "int"]
-) -> pd.DataFrame:
-    """Check if the index_key correctness.
-
-    - Check if the index_key is present in the data frame.
-        (If the index_key is a column in the DataFrame, it is set as the index)
-    - Check if the index_key is of the correct type.
-
-    Args:
-        table_df (pd.DataFrame): The DataFrame to validate.
-        index_key (str): The column name to use as the index of the DataFrame.
-        index_type (str): The type of the index column in the DataFrame.
-            Either 'str' or 'int'. Default is 'int'.
-
-    Returns:
-        pd.DataFrame: The validated DataFrame.
-    """
-    columns = table_df.columns
-    if index_key in columns:
-        table_df = table_df.set_index(index_key)
-
-    if table_df.index.name != index_key:
-        raise NgioTableValidationError(
-            f"index_key: {index_key} not found in data frame"
-        )
-
-    if index_type == "str":
-        if ptypes.is_integer_dtype(table_df.index):
-            # Convert the int index to string is generally safe
-            table_df.index = table_df.index.astype(str)
-
-        if not ptypes.is_string_dtype(table_df.index):
-            raise NgioTableValidationError(
-                f"index_key {index_key} must be of string type"
-            )
-
-    elif index_type == "int":
-        if ptypes.is_string_dtype(table_df.index):
-            # Try to convert the string index to int
-            try:
-                table_df.index = table_df.index.astype(int)
-            except ValueError as e:
-                if "invalid literal for int() with base 10" in str(e):
-                    raise NgioTableValidationError(
-                        f"index_key {index_key} must be of "
-                        "integer type, but found string. We "
-                        "tried implicit conversion failed."
-                    ) from None
-                else:
-                    raise e from e
-
-        if not ptypes.is_integer_dtype(table_df.index):
-            raise NgioTableValidationError(
-                f"index_key {index_key} must be of integer type"
-            )
-
-    else:
-        raise NgioTableValidationError(f"index_type {index_type} not recognized")
-
-    return table_df
-
-
-def validate_table(
-    table_df: pd.DataFrame,
-    index_key: str,
-    index_type: Literal["str", "int"],
-    validators: list[Validator] | None,
-) -> pd.DataFrame:
-    """Validate the table DataFrame.
-
-    - Check if the index_key is present in the data frame.
-        (If the index_key is a column in the DataFrame, it is set as the index)
-    - Check if the index_key is of the correct type.
-    - Apply all provided optional validators.
-
-    Args:
-        table_df (pd.DataFrame): The DataFrame to validate.
-        index_key (str): The column name to use as the index of the DataFrame.
-        index_type (str): The type of the index column in the DataFrame.
-        validators (list[Validator]): A list of functions to further validate table.
-
-    Returns:
-        pd.DataFrame: The validated DataFrame.
-    """
-    table_df = _check_index_key(table_df, index_key, index_type)
-
-    if validators is None:
-        return table_df
-
-    # Apply all provided validators
-    for validator in validators:
-        table_df = validator(table_df)
-
-    return table_df
-
-
-def table_df_to_ad(
-    table_df: pd.DataFrame,
-    index_key: str,
-    index_type: Literal["str", "int"] = "int",
-    validators: list[Validator] | None = None,
-) -> ad.AnnData:
-    """Convert a table DataFrame to an AnnData object.
-
-    Args:
-        table_df (pd.DataFrame): A pandas DataFrame representing a fractal table.
-        index_key (str): The column name to use as the index of the DataFrame.
-        index_type (str): The type of the index column in the DataFrame.
-            Either 'str' or 'int'. Default is 'int'.
-        validators (list[Validator]): A list of functions to further validate the table.
-    """
-    # Check if the index_key is present in the data frame + optional validations
-    table_df = validate_table(
-        table_df=table_df,
-        index_key=index_key,
-        index_type=index_type,
-        validators=validators,
-    )
-
-    # DO NOT SKIP
-    # Convert the index to string ALWAYS to avoid casting issues in AnnData
-    table_df.index = table_df.index.astype(str)
-
-    str_columns, int_columns, num_columns = [], [], []
-    for c_name in table_df.columns:
-        column_df = table_df[c_name]
-        _check_for_mixed_types(column_df)  # Mixed types are not allowed in the table
-        c_type = _check_for_supported_types(
-            column_df
-        )  # Only string and numeric types are allowed
-
-        if c_type == "str":
-            str_columns.append(c_name)
-
-        elif c_type == "int":
-            int_columns.append(c_name)
-
-        elif c_type == "numeric":
-            num_columns.append(c_name)
-
-    # Converting all observations to string
-    obs_df = table_df[str_columns + int_columns]
-    obs_df.index = table_df.index
-
-    x_df = table_df[num_columns]
-
-    if x_df.dtypes.nunique() > 1:
-        x_df = x_df.astype("float64")
-
-    if x_df.empty:
-        # If there are no numeric columns, create an empty array
-        # to avoid AnnData failing to create the object
-        x_df = np.zeros((0, 0), dtype="float64")
-
-    return ad.AnnData(X=x_df, obs=obs_df)
-
-
-def table_ad_to_df(
-    table_ad: ad.AnnData,
-    index_key: str = "label",
-    index_type: Literal["str", "int"] = "int",
-    validators: list[Validator] | None = None,
-    validate_index_name: bool = False,
-) -> pd.DataFrame:
-    """Convert a AnnData object representing a fractal table to a pandas DataFrame.
-
-    Args:
-        table_ad (ad.AnnData): An AnnData object representing a fractal table.
-        index_key (str): The column name to use as the index of the DataFrame.
-            Default is 'label'.
-        index_type (str): The type of the index column in the DataFrame.
-            Either 'str' or 'int'. Default is 'int'.
-        validators (list[Validator]): A list of functions to further validate the table.
-        validate_index_name (bool): If True, the index name is validated.
-    """
-    table_df = table_ad.to_df()
-    table_df[table_ad.obs_keys()] = table_ad.obs
-
-    # Set the index of the DataFrame
-    if index_key in table_df.columns:
-        table_df = table_df.set_index(index_key)
-    elif table_ad.obs.index.name is not None:
-        if validate_index_name:
-            if table_ad.obs.index.name != index_key:
-                raise NgioTableValidationError(
-                    f"Index key {index_key} not found in AnnData object."
-                )
-        table_df.index = table_ad.obs.index
-    elif table_ad.obs.index.name is None:
-        table_df.index = table_ad.obs.index
-        table_df.index.name = index_key
-    else:
-        raise NgioTableValidationError(
-            f"Index key {index_key} not found in AnnData object."
-        )
-
-    table_df = validate_table(
-        table_df=table_df,
-        index_key=index_key,
-        index_type=index_type,
-        validators=validators,
-    )
-    return table_df
-
-
-####################################################################################################
-#
-# Common table validators
-#
-####################################################################################################
-
-
-def validate_columns(
-    table_df: pd.DataFrame,
-    required_columns: list[str],
-    optional_columns: list[str] | None = None,
-) -> pd.DataFrame:
-    """Validate the columns headers of the table.
-
-    If a required column is missing, a TableValidationError is raised.
-    If a list of optional columns is provided, only required and optional columns are
-        allowed in the table.
-
-    Args:
-        table_df (pd.DataFrame): The DataFrame to validate.
-        required_columns (list[str]): A list of required columns.
-        optional_columns (list[str] | None): A list of optional columns.
-            Default is None.
-
-    Returns:
-        pd.DataFrame: The validated DataFrame.
-    """
-    table_header = table_df.columns
-    for column in required_columns:
-        if column not in table_header:
-            raise NgioTableValidationError(
-                f"Could not find required column: {column} in the table"
-            )
-
-    if optional_columns is None:
-        return table_df
-
-    possible_columns = [*required_columns, *optional_columns]
-    for column in table_header:
-        if column not in possible_columns:
-            raise NgioTableValidationError(
-                f"Could not find column: {column} in the list of possible columns. ",
-                f"Possible columns are: {possible_columns}",
-            )
-
-    return table_df
-
-
-def validate_unique_index(table_df: pd.DataFrame) -> pd.DataFrame:
-    """Validate that the index of the table is unique."""
-    if table_df.index.is_unique:
-        return table_df
-
-    # Find the duplicates
-    duplicates = table_df.index[table_df.index.duplicated()].tolist()
-    raise NgioTableValidationError(
-        f"Index of the table contains duplicates values. Duplicate: {duplicates}"
-    )

ngio/tables/tables_group.py

@@ -1,252 +0,0 @@
-"""Module for handling the /tables group in an OME-NGFF file."""
-
-from typing import Literal
-from warnings import warn
-
-import zarr
-from pydantic import ValidationError
-
-from ngio.core.utils import State
-from ngio.io import AccessModeLiteral, StoreLike
-from ngio.tables.v1 import FeatureTableV1, MaskingROITableV1, ROITableV1
-from ngio.utils import ngio_logger
-from ngio.utils._pydantic_utils import BaseWithExtraFields
-
-ROITable = ROITableV1
-IMPLEMENTED_ROI_TABLES = {"1": ROITableV1}
-
-FeatureTable = FeatureTableV1
-IMPLEMENTED_FEATURE_TABLES = {"1": FeatureTableV1}
-
-MaskingROITable = MaskingROITableV1
-IMPLEMENTED_MASKING_ROI_TABLES = {"1": MaskingROITableV1}
-
-Table = ROITable | FeatureTable | MaskingROITable
-
-TableType = Literal["roi_table", "feature_table", "masking_roi_table"]
-
-
-class CommonMeta(BaseWithExtraFields):
-    """Common metadata for all tables."""
-
-    type: TableType
-    fractal_table_version: str = "1"
-
-
-def _find_table_impl(
-    table_type: TableType,
-    version: str,
-) -> Table:
-    """Find the type of table in the group."""
-    if table_type == "roi_table":
-        if version not in IMPLEMENTED_ROI_TABLES:
-            raise ValueError(f"ROI Table version {version} not implemented.")
-        return IMPLEMENTED_ROI_TABLES[version]
-
-    elif table_type == "feature_table":
-        if version not in IMPLEMENTED_FEATURE_TABLES:
-            raise ValueError(f"Feature Table version {version} not implemented.")
-        return IMPLEMENTED_FEATURE_TABLES[version]
-
-    elif table_type == "masking_roi_table":
-        if version not in IMPLEMENTED_MASKING_ROI_TABLES:
-            raise ValueError(f"Masking ROI Table version {version} not implemented.")
-        return IMPLEMENTED_MASKING_ROI_TABLES[version]
-
-    else:
-        raise ValueError(f"Table type {table_type} not recognized.")
-
-
-def _get_table_impl(
-    group: zarr.Group,
-    name: str,
-    validate_metadata: bool = True,
-    table_type: TableType | None = None,
-    validate_table: bool = True,
-    index_key: str | None = None,
-) -> Table:
-    if validate_metadata:
-        common_meta = CommonMeta(**group.attrs)
-        table_type = common_meta.type
-    else:
-        common_meta = CommonMeta.model_construct(**group.attrs)
-        if table_type is None:
-            raise ValueError(
-                "Table type must be provided if metadata is not validated."
-            )
-
-    version = common_meta.fractal_table_version
-    return _find_table_impl(table_type=table_type, version=version)(
-        group=group,
-        name=name,
-        validate_metadata=validate_metadata,
-        validate_table=validate_table,
-        index_key=index_key,
-    )
-
-
-class TableGroup:
-    """A class to handle the /labels group in an OME-NGFF file."""
-
-    def __init__(
-        self, group: StoreLike | zarr.Group, mode: AccessModeLiteral = "r+"
-    ) -> None:
-        """Initialize the LabelGroupHandler."""
-        self._mode = mode
-        if not isinstance(group, zarr.Group):
-            group = zarr.open_group(group, mode=self._mode)
-
-        table_group = group.get("tables", None)
-
-        if table_group is None and not group.read_only:
-            table_group = group.create_group("tables")
-            table_group.attrs["tables"] = []
-
-        assert isinstance(table_group, zarr.Group) or table_group is None
-        self._table_group = table_group
-
-        if table_group is None or table_group.read_only:
-            self._state = State.MEMORY
-        else:
-            self._state = State.CONSOLIDATED
-
-        self._virtual_tables: list[Table] = []
-
-    def _validate_list_of_tables(self, list_of_tables: list[str]) -> None:
-        """Validate the list of tables.
-
-        Args:
-            list_of_tables (list[str]): The list of tables to validate.
-        """
-        if self._table_group is None:
-            return None
-
-        for table_name in list_of_tables:
-            table = self._table_group.get(table_name, None)
-            if table is None:
-                ngio_logger.warning(
-                    f"Table {table_name} not found in the group. "
-                    "Consider removing it from the list of tables."
-                )
-
-    def _get_list_of_tables(self) -> list[str]:
-        """Return the list of tables."""
-        if self._table_group is None:
-            return []
-
-        list_of_tables = self._table_group.attrs.get("tables", [])
-        self._validate_list_of_tables(list_of_tables)
-        assert isinstance(list_of_tables, list)
-        assert all(isinstance(table_name, str) for table_name in list_of_tables)
-        return list_of_tables
-
-    def list(
-        self,
-        table_type: TableType | None = None,
-    ) -> list[str]:
-        """List all labels in the group.
-
-        Args:
-            table_type (str): The type of table to list.
-                If None, all tables are listed.
-                Allowed values are: 'roi_table', 'feature_table', 'masking_roi_table'.
-        """
-        if self._table_group is None:
-            return []
-
-        list_of_tables = self._get_list_of_tables()
-        self._validate_list_of_tables(list_of_tables=list_of_tables)
-
-        all_table_types = ["roi_table", "feature_table", "masking_roi_table"]
-
-        if table_type is None:
-            return list_of_tables
-
-        else:
-            if table_type not in all_table_types:
-                raise ValueError(
-                    f"Table type {table_type} not recognized. "
-                    f" Allowed values are: {all_table_types}"
-                )
-            list_of_typed_tables = []
-            for table_name in list_of_tables:
-                table = self._table_group[table_name]
-                try:
-                    common_meta = CommonMeta(**table.attrs)
-                    if common_meta.type == table_type:
-                        list_of_typed_tables.append(table_name)
-                except ValidationError:
-                    warn(
-                        f"Table {table_name} metadata is not correctly formatted.",
-                        stacklevel=1,
-                    )
-            return list_of_typed_tables
-
-    def get_table(
-        self,
-        name: str,
-        table_type: TableType | None = None,
-        validate_metadata: bool = True,
-        validate_table: bool = True,
-        index_key: str | None = None,
-    ) -> Table:
-        """Get a label from the group.
-
-        Args:
-            name (str): The name of the table to get.
-            table_type (str): The type of table to get.
-                If None, all the table type will be inferred from the metadata.
-                Allowed values are: 'roi_table', 'feature_table', 'masking_roi_table'.
-            validate_metadata (bool): Whether to validate the metadata of the table.
-            validate_table (bool): Whether to validate the table.
-            index_key (str): The column name to use as the index of the DataFrame.
-                This is usually defined in the metadata of the table, if given here,
-                it will overwrite the metadata.
-        """
-        if self._table_group is None:
-            raise ValueError("No tables group found in the group.")
-
-        list_of_tables = self._get_list_of_tables()
-        if name not in list_of_tables:
-            raise ValueError(f"Table {name} not found in the group.")
-
-        return _get_table_impl(
-            group=self._table_group[name],
-            name=name,
-            validate_metadata=validate_metadata,
-            table_type=table_type,
-            validate_table=validate_table,
-            index_key=index_key,
-        )
-
-    def new(
-        self,
-        name: str,
-        table_type: TableType = "roi_table",
-        overwrite: bool = False,
-        version: str = "1",
-        **type_specific_kwargs: dict,
-    ) -> Table:
-        """Add a new table to the group."""
-        if self._table_group is None:
-            raise ValueError("No tables group found in the group.")
-
-        list_of_tables = self._get_list_of_tables()
-        if not overwrite and name in list_of_tables:
-            raise ValueError(f"Table {name} already exists in the group.")
-
-        if overwrite and name in list_of_tables:
-            list_of_tables.remove(name)
-
-        table_impl = _find_table_impl(table_type=table_type, version=version)
-        new_table = table_impl._new(
-            parent_group=self._table_group,
-            name=name,
-            overwrite=overwrite,
-            **type_specific_kwargs,
-        )
-
-        self._table_group.attrs["tables"] = [*list_of_tables, name]
-
-        assert isinstance(new_table, ROITable | FeatureTable | MaskingROITable)
-        return new_table