ngio 0.5.0__py3-none-any.whl → 0.5.0a2__py3-none-any.whl

ngio/images/_create_utils.py

@@ -1,406 +0,0 @@
-"""Utility functions for working with OME-Zarr images."""
-
-import warnings
-from collections.abc import Mapping, Sequence
-from typing import Any, Literal, TypeVar
-
-from zarr.core.array import CompressorLike
-
-from ngio.common._pyramid import ChunksLike, ImagePyramidBuilder, ShardsLike
-from ngio.ome_zarr_meta import (
-    NgioImageMeta,
-    NgioLabelMeta,
-    update_ngio_meta,
-)
-from ngio.ome_zarr_meta.ngio_specs import (
-    AxesHandler,
-    Channel,
-    ChannelsMeta,
-    DefaultNgffVersion,
-    DefaultSpaceUnit,
-    DefaultTimeUnit,
-    NgffVersions,
-    SpaceUnits,
-    TimeUnits,
-    build_axes_handler,
-    build_canonical_axes_handler,
-    canonical_axes_order,
-    canonical_label_axes_order,
-)
-from ngio.ome_zarr_meta.ngio_specs._axes import AxesSetup
-from ngio.utils import NgioValueError, StoreOrGroup, ZarrGroupHandler
-
-_image_or_label_meta = TypeVar("_image_or_label_meta", NgioImageMeta, NgioLabelMeta)
-
-
-def _align_to_axes(
-    *,
-    values: dict[str, float],
-    axes_handler: AxesHandler,
-    default_value: float = 1.0,
-) -> tuple[float, ...]:
-    """Align given values to axes names."""
-    aligned_values = [default_value] * len(axes_handler.axes_names)
-    for ax, value in values.items():
-        index = axes_handler.get_index(ax)
-        if index is not None:
-            aligned_values[index] = value
-    return tuple(aligned_values)
-
-
-def _check_deprecated_scaling_factors(
-    *,
-    yx_scaling_factor: float | tuple[float, float] | None = None,
-    z_scaling_factor: float | None = None,
-    scaling_factors: Sequence[float] | Literal["auto"] = "auto",
-    shape: tuple[int, ...],
-) -> Sequence[float] | Literal["auto"]:
-    if yx_scaling_factor is not None or z_scaling_factor is not None:
-        warnings.warn(
-            "The 'yx_scaling_factor' and 'z_scaling_factor' arguments are deprecated "
-            "and will be removed in ngio=0.6. Please use the 'scaling_factors' "
-            "argument instead.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        if scaling_factors != "auto":
-            raise NgioValueError(
-                "Cannot use both 'scaling_factors' and deprecated "
-                "'yx_scaling_factor'/'z_scaling_factor' arguments."
-            )
-        if isinstance(yx_scaling_factor, tuple):
-            if len(yx_scaling_factor) != 2:
-                raise NgioValueError(
-                    "yx_scaling_factor tuple must have length 2 for y and x scaling."
-                )
-            y_scale = yx_scaling_factor[0]
-            x_scale = yx_scaling_factor[1]
-        else:
-            y_scale = yx_scaling_factor if yx_scaling_factor is not None else 2.0
-            x_scale = yx_scaling_factor if yx_scaling_factor is not None else 2.0
-        z_scale = z_scaling_factor if z_scaling_factor is not None else 1.0
-        scaling_factors = (z_scale, x_scale, y_scale)
-        if len(scaling_factors) < len(shape):
-            padding = (1.0,) * (len(shape) - len(scaling_factors))
-            scaling_factors = padding + scaling_factors
-
-        return scaling_factors
-    return scaling_factors
-
-
-def _compute_scaling_factors(
-    *,
-    scaling_factors: Sequence[float] | Literal["auto"],
-    shape: tuple[int, ...],
-    axes_handler: AxesHandler,
-    xy_scaling_factor: float | tuple[float, float] | None = None,
-    z_scaling_factor: float | None = None,
-) -> tuple[float, ...]:
-    """Compute scaling factors for given axes names."""
-    # TODO remove with ngio 0.6
-    scaling_factors = _check_deprecated_scaling_factors(
-        yx_scaling_factor=xy_scaling_factor,
-        z_scaling_factor=z_scaling_factor,
-        scaling_factors=scaling_factors,
-        shape=shape,
-    )
-    if scaling_factors == "auto":
-        return _align_to_axes(
-            values={
-                "x": 2.0,
-                "y": 2.0,
-                "z": 1.0,
-            },
-            axes_handler=axes_handler,
-        )
-    if len(scaling_factors) != len(shape):
-        raise NgioValueError(
-            "Length of scaling_factors does not match the number of dimensions."
-        )
-    return tuple(scaling_factors)
-
-
-def compute_base_scale(
-    *,
-    pixelsize: float | tuple[float, float],
-    z_spacing: float,
-    time_spacing: float,
-    axes_handler: AxesHandler,
-) -> tuple[float, ...]:
-    """Compute base scale for given axes names."""
-    if isinstance(pixelsize, tuple):
-        if len(pixelsize) != 2:
-            raise NgioValueError(
-                "pixelsize tuple must have length 2 for y and x pixel sizes."
-            )
-        x_size = pixelsize[1]
-        y_size = pixelsize[0]
-    else:
-        x_size = pixelsize
-        y_size = pixelsize
-    return _align_to_axes(
-        values={
-            "x": x_size,
-            "y": y_size,
-            "z": z_spacing,
-            "t": time_spacing,
-        },
-        axes_handler=axes_handler,
-    )
-
-
-def _create_image_like_group(
-    *,
-    store: StoreOrGroup,
-    pyramid_builder: ImagePyramidBuilder,
-    meta: _image_or_label_meta,
-    overwrite: bool = False,
-) -> ZarrGroupHandler:
-    """Advanced create empty image container function placeholder."""
-    mode = "w" if overwrite else "w-"
-    group_handler = ZarrGroupHandler(
-        store=store, mode=mode, cache=False, zarr_format=meta.zarr_format
-    )
-    update_ngio_meta(group_handler, meta)
-    # Reopen in r+ mode
-    group_handler = group_handler.reopen_handler()
-    # Write the pyramid
-    pyramid_builder.to_zarr(group=group_handler.group)
-    return group_handler
-
-
-def _add_channels_meta(
-    *,
-    meta: _image_or_label_meta,
-    channels_meta: Sequence[str | Channel] | None = None,
-) -> _image_or_label_meta:
-    """Create ChannelsMeta from given channels_meta input."""
-    if isinstance(meta, NgioLabelMeta):
-        if channels_meta is not None:
-            raise NgioValueError(
-                "Cannot add channels_meta to NgioLabelMeta. "
-                "Labels do not have channels."
-            )
-        else:
-            return meta
-    if channels_meta is None:
-        return meta
-    list_of_channels = []
-    for c in channels_meta:
-        if isinstance(c, str):
-            channel = Channel.default_init(label=c)
-        elif isinstance(c, Channel):
-            channel = c
-        else:
-            raise NgioValueError(
-                "channels_meta must be a list of strings or Channel objects."
-            )
-        list_of_channels.append(channel)
-
-    channels_meta_ = ChannelsMeta(channels=list_of_channels)
-    meta.set_channels_meta(channels_meta=channels_meta_)
-    return meta
-
-
-def _build_axes_handler(
-    *,
-    shape: tuple[int, ...],
-    meta_type: type[_image_or_label_meta],
-    axes_names: Sequence[str] | None = None,
-    axes_setup: AxesSetup | None = None,
-) -> AxesHandler:
-    """Build axes handler for given shape and axes names."""
-    if meta_type is NgioImageMeta:
-        canonical_axes_order_ = canonical_axes_order()
-    else:
-        canonical_axes_order_ = canonical_label_axes_order()
-    if axes_names is None:
-        axes_names = canonical_axes_order_[-len(shape) :]
-
-    if axes_setup is None:
-        return build_canonical_axes_handler(
-            axes_names=axes_names,
-            canonical_channel_order=canonical_axes_order_,
-        )
-    return build_axes_handler(
-        axes_names=axes_names,
-        axes_setup=axes_setup,
-    )
-
-
-def init_image_like(
-    *,
-    # Where to create the image
-    store: StoreOrGroup,
-    # Ngff image parameters
-    meta_type: type[_image_or_label_meta],
-    shape: Sequence[int],
-    pixelsize: float | tuple[float, float],
-    z_spacing: float = 1.0,
-    time_spacing: float = 1.0,
-    scaling_factors: Sequence[float] | Literal["auto"] = "auto",
-    levels: int | list[str] = 5,
-    translation: Sequence[float] | None = None,
-    space_unit: SpaceUnits | str | None = DefaultSpaceUnit,
-    time_unit: TimeUnits | str | None = DefaultTimeUnit,
-    axes_names: Sequence[str] | None = None,
-    name: str | None = None,
-    channels_meta: Sequence[str | Channel] | None = None,
-    ngff_version: NgffVersions = DefaultNgffVersion,
-    # Zarr Array parameters
-    chunks: ChunksLike = "auto",
-    shards: ShardsLike | None = None,
-    dtype: str = "uint16",
-    dimension_separator: Literal[".", "/"] = "/",
-    compressors: CompressorLike = "auto",
-    extra_array_kwargs: Mapping[str, Any] | None = None,
-    # internal axes configuration for advanced use cases
-    axes_setup: AxesSetup | None = None,
-    # Whether to overwrite existing image
-    overwrite: bool = False,
-    # Deprecated arguments
-    yx_scaling_factor: float | tuple[float, float] | None = None,
-    z_scaling_factor: float | None = None,
-) -> tuple[ZarrGroupHandler, AxesSetup]:
-    """Create an empty OME-Zarr image with the given shape and metadata."""
-    shape = tuple(shape)
-    axes_handler = _build_axes_handler(
-        shape=shape,
-        meta_type=meta_type,
-        axes_names=axes_names,
-        axes_setup=axes_setup,
-    )
-    if len(shape) != len(axes_handler.axes_names):
-        raise NgioValueError(
-            f"Mismatch between shape {shape} "
-            f"and number of axes {len(axes_handler.axes_names)}."
-        )
-    base_scale = compute_base_scale(
-        pixelsize=pixelsize,
-        z_spacing=z_spacing,
-        time_spacing=time_spacing,
-        axes_handler=axes_handler,
-    )
-    scaling_factors = _compute_scaling_factors(
-        scaling_factors=scaling_factors,
-        shape=shape,
-        axes_handler=axes_handler,
-        xy_scaling_factor=yx_scaling_factor,
-        z_scaling_factor=z_scaling_factor,
-    )
-    if isinstance(levels, int):
-        levels_paths = tuple(str(i) for i in range(levels))
-    else:
-        levels_paths = tuple(levels)
-
-    pyramid_builder = ImagePyramidBuilder.from_scaling_factors(
-        levels_paths=levels_paths,
-        scaling_factors=scaling_factors,
-        base_shape=shape,
-        base_scale=base_scale,
-        base_translation=translation,
-        axes=axes_handler.axes_names,
-        chunks=chunks,
-        data_type=dtype,
-        dimension_separator=dimension_separator,
-        compressors=compressors,
-        shards=shards,
-        zarr_format=2 if ngff_version == "0.4" else 3,
-        other_array_kwargs=extra_array_kwargs,
-    )
-    meta = meta_type.default_init(
-        levels=[p.path for p in pyramid_builder.levels],
-        axes_handler=axes_handler,
-        scales=[p.scale for p in pyramid_builder.levels],
-        translations=[p.translation for p in pyramid_builder.levels],
-        name=name,
-        version=ngff_version,
-    )
-    meta = _add_channels_meta(meta=meta, channels_meta=channels_meta)
-    # Keep this creation at the end to avoid partial creations on errors
-    image_handler = _create_image_like_group(
-        store=store,
-        pyramid_builder=pyramid_builder,
-        meta=meta,
-        overwrite=overwrite,
-    )
-    return image_handler, axes_handler.axes_setup
-
-
-def init_image_like_from_shapes(
-    *,
-    # Where to create the image
-    store: StoreOrGroup,
-    # Ngff image parameters
-    meta_type: type[_image_or_label_meta],
-    shapes: Sequence[tuple[int, ...]],
-    base_scale: tuple[float, ...] | list[tuple[float, ...]],
-    levels: list[str] | None = None,
-    translation: Sequence[float] | None = None,
-    space_unit: SpaceUnits | str | None = DefaultSpaceUnit,
-    time_unit: TimeUnits | str | None = DefaultTimeUnit,
-    axes_names: Sequence[str] | None = None,
-    name: str | None = None,
-    channels_meta: Sequence[str | Channel] | None = None,
-    ngff_version: NgffVersions = DefaultNgffVersion,
-    # Zarr Array parameters
-    chunks: ChunksLike = "auto",
-    shards: ShardsLike | None = None,
-    dtype: str = "uint16",
-    dimension_separator: Literal[".", "/"] = "/",
-    compressors: CompressorLike = "auto",
-    extra_array_kwargs: Mapping[str, Any] | None = None,
-    # internal axes configuration for advanced use cases
-    axes_setup: AxesSetup | None = None,
-    # Whether to overwrite existing image
-    overwrite: bool = False,
-) -> tuple[ZarrGroupHandler, AxesSetup]:
-    """Create an empty OME-Zarr image with the given shape and metadata."""
-    base_shape = shapes[0]
-    axes_handler = _build_axes_handler(
-        shape=base_shape,
-        meta_type=meta_type,
-        axes_names=axes_names,
-        axes_setup=axes_setup,
-    )
-    if len(base_shape) != len(axes_handler.axes_names):
-        raise NgioValueError(
-            f"Mismatch between shape {base_shape} "
-            f"and number of axes {len(axes_handler.axes_names)}."
-        )
-    if levels is None:
-        levels_paths = tuple(str(i) for i in range(len(shapes)))
-    else:
-        levels_paths = tuple(levels)
-
-    pyramid_builder = ImagePyramidBuilder.from_shapes(
-        shapes=shapes,
-        base_scale=base_scale,
-        base_translation=translation,
-        levels_paths=levels_paths,
-        axes=axes_handler.axes_names,
-        chunks=chunks,
-        data_type=dtype,
-        dimension_separator=dimension_separator,
-        compressors=compressors,
-        shards=shards,
-        zarr_format=2 if ngff_version == "0.4" else 3,
-        other_array_kwargs=extra_array_kwargs,
-    )
-    meta = meta_type.default_init(
-        levels=[p.path for p in pyramid_builder.levels],
-        axes_handler=axes_handler,
-        scales=[p.scale for p in pyramid_builder.levels],
-        translations=[p.translation for p in pyramid_builder.levels],
-        name=name,
-        version=ngff_version,
-    )
-    meta = _add_channels_meta(meta=meta, channels_meta=channels_meta)
-    # Keep this creation at the end to avoid partial creations on errors
-    image_handler = _create_image_like_group(
-        store=store,
-        pyramid_builder=pyramid_builder,
-        meta=meta,
-        overwrite=overwrite,
-    )
-    return image_handler, axes_handler.axes_setup

ngio/tables/backends/_py_arrow_backends.py

@@ -1,222 +0,0 @@
-from typing import Literal
-
-import polars as pl
-import pyarrow as pa
-import pyarrow.csv as pa_csv
-import pyarrow.dataset as pa_ds
-import pyarrow.fs as pa_fs
-import pyarrow.parquet as pa_parquet
-from pandas import DataFrame
-from polars import DataFrame as PolarsDataFrame
-from polars import LazyFrame
-from zarr.storage import FsspecStore, LocalStore, MemoryStore, ZipStore
-
-from ngio.tables.backends._abstract_backend import AbstractTableBackend
-from ngio.tables.backends._utils import normalize_pandas_df, normalize_polars_lf
-from ngio.utils import NgioValueError
-from ngio.utils._zarr_utils import _make_sync_fs
-
-
-class PyArrowBackend(AbstractTableBackend):
-    """A class to load and write small tables in CSV format."""
-
-    def __init__(
-        self,
-        table_name: str,
-        table_format: Literal["csv", "parquet"] = "parquet",
-    ):
-        self.table_name = table_name
-        self.table_format = table_format
-
-    @staticmethod
-    def implements_anndata() -> bool:
-        """Whether the handler implements the anndata protocol."""
-        return False
-
-    @staticmethod
-    def implements_pandas() -> bool:
-        """Whether the handler implements the dataframe protocol."""
-        return True
-
-    @staticmethod
-    def implements_polars() -> bool:
-        """Whether the handler implements the polars protocol."""
-        return True
-
-    @staticmethod
-    def backend_name() -> str:
-        """Return the name of the backend."""
-        raise NotImplementedError(
-            "The backend_name method must be implemented in the subclass."
-        )
-
-    def _raise_store_type_not_supported(self):
-        """Raise an error for unsupported store types."""
-        ext = self.table_name.split(".")[-1]
-        store = self._group_handler.store
-        raise NgioValueError(
-            f"Ngio does not support reading a {ext} table from a "
-            f"store of type {type(store)}. "
-            "Please make sure to use a compatible "
-            "store like a LocalStore, or "
-            "FsspecStore, or MemoryStore, or ZipStore."
-        )
-
-    def _load_from_local_store(self, store: LocalStore, path: str) -> pa_ds.Dataset:
-        """Load the table from a directory store."""
-        root_path = store.root
-        table_path = f"{root_path}/{path}/{self.table_name}"
-        dataset = pa_ds.dataset(table_path, format=self.table_format)
-        return dataset
-
-    def _load_from_fsspec_store(self, store: FsspecStore, path: str) -> pa_ds.Dataset:
-        """Load the table from an FS store."""
-        table_path = f"{store.path}/{path}/{self.table_name}"
-        fs = _make_sync_fs(store.fs)
-        dataset = pa_ds.dataset(table_path, format=self.table_format, filesystem=fs)
-        return dataset
-
-    def _load_from_in_memory_store(
-        self, store: MemoryStore, path: str
-    ) -> pa_ds.Dataset:
-        """Load the table from an in-memory store."""
-        table_path = f"{path}/{self.table_name}"
-        table = store._store_dict.get(table_path, None)
-        if table is None:
-            raise NgioValueError(
-                f"Table {self.table_name} not found in the in-memory store at "
-                f"path {path}."
-            )
-        assert isinstance(table, pa.Table)
-        dataset = pa_ds.dataset(table)
-        return dataset
-
-    def _load_from_zip_store(self, store: ZipStore, path: str) -> pa_ds.Dataset:
-        """Load the table from a zip store."""
-        raise NotImplementedError("Zip store loading is not implemented yet.")
-
-    def _load_pyarrow_dataset(self) -> pa_ds.Dataset:
-        """Load the table as a pyarrow Dataset."""
-        store = self._group_handler.store
-        path = self._group_handler.group.path
-        if isinstance(store, LocalStore):
-            return self._load_from_local_store(store, path)
-        elif isinstance(store, FsspecStore):
-            return self._load_from_fsspec_store(store, path)
-        elif isinstance(store, MemoryStore):
-            return self._load_from_in_memory_store(store, path)
-        elif isinstance(store, ZipStore):
-            return self._load_from_zip_store(store, path)
-        self._raise_store_type_not_supported()
-
-    def load_as_pandas_df(self) -> DataFrame:
-        """Load the table as a pandas DataFrame."""
-        dataset = self._load_pyarrow_dataset()
-        dataframe = dataset.to_table().to_pandas()
-        dataframe = normalize_pandas_df(
-            dataframe,
-            index_key=self.index_key,
-            index_type=self.index_type,
-            reset_index=False,
-        )
-        return dataframe
-
-    def load(self) -> DataFrame:
-        """Load the table as a pandas DataFrame."""
-        return self.load_as_pandas_df()
-
-    def load_as_polars_lf(self) -> LazyFrame:
-        """Load the table as a polars LazyFrame."""
-        dataset = self._load_pyarrow_dataset()
-        lazy_frame = pl.scan_pyarrow_dataset(dataset)
-        if not isinstance(lazy_frame, LazyFrame):
-            raise NgioValueError(
-                "Table is not a lazy frame. Please report this issue as an ngio bug."
-                f" {type(lazy_frame)}"
-            )
-
-        lazy_frame = normalize_polars_lf(
-            lazy_frame,
-            index_key=self.index_key,
-            index_type=self.index_type,
-        )
-        return lazy_frame
-
-    def _write_to_stream(self, stream, table: pa.Table) -> None:
-        """Write the table to a stream."""
-        if self.table_format == "parquet":
-            pa_parquet.write_table(table, stream)
-        elif self.table_format == "csv":
-            pa_csv.write_csv(table, stream)
-        else:
-            raise NgioValueError(
-                f"Unsupported table format: {self.table_format}. "
-                "Supported formats are 'parquet' and 'csv'."
-            )
-
-    def _write_to_local_store(
-        self, store: LocalStore, path: str, table: pa.Table
-    ) -> None:
-        """Write the table to a directory store."""
-        root_path = store.root
-        table_path = f"{root_path}/{path}/{self.table_name}"
-        self._write_to_stream(table_path, table)
-
-    def _write_to_fsspec_store(
-        self, store: FsspecStore, path: str, table: pa.Table
-    ) -> None:
-        """Write the table to an FS store."""
-        table_path = f"{store.path}/{path}/{self.table_name}"
-        fs = _make_sync_fs(store.fs)
-        fs = pa_fs.PyFileSystem(pa_fs.FSSpecHandler(fs))
-        with fs.open_output_stream(table_path) as out_stream:
-            self._write_to_stream(out_stream, table)
-
-    def _write_to_in_memory_store(
-        self, store: MemoryStore, path: str, table: pa.Table
-    ) -> None:
-        """Write the table to an in-memory store."""
-        table_path = f"{path}/{self.table_name}"
-        store._store_dict[table_path] = table
-
-    def _write_to_zip_store(self, store: ZipStore, path: str, table: pa.Table) -> None:
-        """Write the table to a zip store."""
-        raise NotImplementedError("Writing to zip store is not implemented yet.")
-
-    def _write_pyarrow_dataset(self, dataset: pa.Table) -> None:
-        """Write the table from a pyarrow Dataset."""
-        store = self._group_handler.store
-        path = self._group_handler.group.path
-        if isinstance(store, LocalStore):
-            return self._write_to_local_store(store=store, path=path, table=dataset)
-        elif isinstance(store, FsspecStore):
-            return self._write_to_fsspec_store(store=store, path=path, table=dataset)
-        elif isinstance(store, MemoryStore):
-            return self._write_to_in_memory_store(store=store, path=path, table=dataset)
-        elif isinstance(store, ZipStore):
-            return self._write_to_zip_store(store=store, path=path, table=dataset)
-        self._raise_store_type_not_supported()
-
-    def write_from_pandas(self, table: DataFrame) -> None:
-        """Write the table from a pandas DataFrame."""
-        table = normalize_pandas_df(
-            table,
-            index_key=self.index_key,
-            index_type=self.index_type,
-            reset_index=True,
-        )
-        table = pa.Table.from_pandas(table, preserve_index=False)
-        self._write_pyarrow_dataset(table)
-
-    def write_from_polars(self, table: PolarsDataFrame | LazyFrame) -> None:
-        """Write the table from a polars DataFrame or LazyFrame."""
-        table = normalize_polars_lf(
-            table,
-            index_key=self.index_key,
-            index_type=self.index_type,
-        )
-
-        if isinstance(table, LazyFrame):
-            table = table.collect()
-        table = table.to_arrow()
-        self._write_pyarrow_dataset(table)