dataeval-flow 0.1.0__py3-none-any.whl

dataeval_flow/embeddings.py

@@ -0,0 +1,135 @@
+"""Embeddings convenience builder wrapping DataEval."""
+
+__all__ = [
+    "build_embeddings",
+    "build_extractor",
+]
+
+import logging
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+from dataeval import Embeddings
+from dataeval.extractors import BoVWExtractor, FlattenExtractor, OnnxExtractor, TorchExtractor
+from dataeval.protocols import AnnotatedDataset
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from dataeval_flow.config.schemas import ExtractorConfig
+
+
+def build_embeddings(
+    dataset: AnnotatedDataset[Any],
+    extractor_config: "ExtractorConfig",
+    transforms: Callable | None = None,
+    batch_size: int | None = None,
+) -> Embeddings:
+    """Build Embeddings from dataset and extractor config.
+
+    Creates the appropriate extractor based on the config's model type and
+    wraps it in a DataEval Embeddings instance.
+
+    Parameters
+    ----------
+    dataset : MaiteDataset
+        Input dataset.
+    extractor_config : ExtractorConfig
+        Extractor configuration with model type and params.
+    transforms : Callable | None
+        Preprocessing transforms to apply before encoding.
+        Only used by extractor types that accept it (onnx, torch, uncertainty).
+
+    Returns
+    -------
+    Embeddings
+        DataEval Embeddings instance (implements FeatureExtractor).
+    """
+
+    extractor = build_extractor(extractor_config, transforms)
+    return Embeddings(dataset, extractor=extractor, batch_size=batch_size)
+
+
+def build_extractor(extractor_config: "ExtractorConfig", transforms: Callable | None = None) -> Callable:
+    """Build a standalone extractor (not wrapped in Embeddings).
+
+    Used for workflows that need to apply the extractor separately from embedding extraction
+    (e.g. to extract metadata features for evaluation).
+
+    Parameters
+    ----------
+    extractor_config : ExtractorConfig
+        Extractor configuration with model type and params.
+    transforms : Callable | None
+        Preprocessing transforms to apply before encoding.
+        Only used by extractor types that accept it (onnx, torch, uncertainty).
+
+    Returns
+    -------
+    Callable
+        A callable extractor function that takes a dataset and returns extracted features.
+    """
+    from dataeval_flow.config.schemas._extractor import (
+        BoVWExtractorConfig,
+        FlattenExtractorConfig,
+        OnnxExtractorConfig,
+        TorchExtractorConfig,
+    )
+
+    logger.debug("Building %s extractor", extractor_config.model)
+
+    if isinstance(extractor_config, OnnxExtractorConfig):
+        extractor = OnnxExtractor(
+            extractor_config.model_path,
+            transforms=transforms,
+            output_name=extractor_config.output_name,
+            flatten=extractor_config.flatten,
+        )
+    elif isinstance(extractor_config, BoVWExtractorConfig):
+        extractor = BoVWExtractor(vocab_size=extractor_config.vocab_size)
+    elif isinstance(extractor_config, FlattenExtractorConfig):
+        extractor = FlattenExtractor()
+    elif isinstance(extractor_config, TorchExtractorConfig):
+        extractor = _build_torch_extractor(extractor_config, transforms)
+    else:
+        raise ValueError(
+            f"Extractor type '{extractor_config.model}' is not yet implemented. "
+            f"Currently supported: onnx, bovw, flatten, torch."
+        )
+    return extractor
+
+
+def _build_torch_extractor(
+    config: Any,
+    transforms: Callable | None = None,
+) -> TorchExtractor:
+    """Build a TorchExtractor from config, loading the model from disk.
+
+    The *transforms* from ``build_preprocessing`` is a numpy→numpy wrapper.
+    ``TorchExtractor`` handles tensor conversion internally and expects raw
+    torchvision transforms, so we unwrap the ``v2.Compose`` when possible.
+    """
+    import torch
+    from torchvision.transforms import v2
+
+    # Unwrap the numpy wrapper produced by build_preprocessing to get the
+    # raw v2.Compose that TorchExtractor expects (it handles tensor
+    # conversion internally via torch.as_tensor).
+    torch_transforms: v2.Compose | Callable | None = None
+    if transforms is not None:
+        inner = getattr(transforms, "__wrapped__", None)
+        if isinstance(inner, v2.Compose):
+            torch_transforms = inner
+        elif isinstance(transforms, v2.Compose):
+            torch_transforms = transforms
+        else:
+            torch_transforms = transforms
+
+    model = torch.load(config.model_path, map_location="cpu", weights_only=False)
+    return TorchExtractor(
+        model,
+        transforms=torch_transforms,
+        device=config.device,
+        layer_name=config.layer_name,
+        use_output=config.use_output,
+    )

dataeval_flow/metadata.py

@@ -0,0 +1,48 @@
+"""Metadata convenience builder wrapping DataEval."""
+
+__all__ = ["build_metadata"]
+
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any
+
+from dataeval import Metadata
+from dataeval.protocols import AnnotatedDataset
+
+if TYPE_CHECKING:
+    from dataeval_flow.config.schemas import AutoBinMethod
+
+
+def build_metadata(
+    dataset: AnnotatedDataset[Any],
+    auto_bin_method: "AutoBinMethod | None" = None,
+    exclude: Sequence[str] | None = None,
+    continuous_factor_bins: Mapping[str, int | Sequence[float]] | None = None,
+) -> Metadata:
+    """Build Metadata from dataset and config.
+
+    Parameters
+    ----------
+    dataset : AnnotatedDataset
+        Input dataset.
+    auto_bin_method : AutoBinMethod | None
+        Method for automatic binning of continuous values.
+    exclude : list[str] | None
+        Metadata columns to exclude.
+    continuous_factor_bins : dict[str, int | list[float]] | None
+        Number of uniform bins (int) or explicit bin edges (list[float])
+        for specific continuous factors.
+
+    Returns
+    -------
+    Metadata
+        DataEval Metadata instance.
+    """
+    kwargs = {}
+    if auto_bin_method is not None:
+        kwargs["auto_bin_method"] = auto_bin_method
+    if exclude is not None:
+        kwargs["exclude"] = exclude
+    if continuous_factor_bins is not None:
+        kwargs["continuous_factor_bins"] = continuous_factor_bins
+
+    return Metadata(dataset, **kwargs)

dataeval_flow/preprocessing.py

@@ -0,0 +1,141 @@
+"""Preprocessing utilities for image transforms.
+
+Provides configuration-driven preprocessing using torchvision.transforms.v2.
+Any v2 transform can be specified by name in YAML config.
+
+The returned callable accepts a numpy CHW array, converts to a torch tensor
+for torchvision transforms, then converts back to a numpy CHW array so that
+ONNX extractors receive the format they expect.
+
+Example YAML:
+    preprocessing:
+      - step: Resize
+        params: {size: [256, 256], antialias: true}
+      - step: Normalize
+        params: {mean: [0.485, 0.456, 0.406], std: [0.229, 0.224, 0.225]}
+
+Example Python:
+    >>> from dataeval_flow.preprocessing import PreprocessingStep, build_preprocessing
+    >>> steps = [PreprocessingStep(step="Resize", params={"size": 256})]
+    >>> transform = build_preprocessing(steps)
+    >>> output = transform(input_array)  # numpy CHW -> numpy CHW
+"""
+
+__all__ = ["PreprocessingStep", "build_preprocessing"]
+
+
+import logging
+from collections.abc import Sequence
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+from pydantic import BaseModel, Field
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class PreprocessingStep(BaseModel):
+    """Single preprocessing step.
+
+    Pass-through to torchvision.transforms.v2 - any transform name is allowed.
+    See: https://pytorch.org/vision/stable/transforms.html
+
+    Example
+    -------
+    >>> step = PreprocessingStep(step="Resize", params={"size": 256, "antialias": True})
+    >>> step = PreprocessingStep(step="Normalize", params={"mean": [0.485], "std": [0.229]})
+    """
+
+    step: str = Field(description="Transform name from torchvision.transforms.v2")
+    params: dict[str, Any] = Field(default_factory=dict)
+
+
+class _PreprocessingTransform:
+    """Callable wrapper around ``v2.Compose`` with a stable ``repr``.
+
+    The default ``repr`` of a closure includes the memory address which
+    changes every run, causing unnecessary cache misses when the cache
+    key is derived from ``repr(transforms)``.  This wrapper delegates to
+    ``v2.Compose.__repr__`` which is deterministic.
+    """
+
+    __slots__ = ("__wrapped__",)
+    __wrapped__: Any
+
+    def __init__(self, wrapped: Any) -> None:
+        self.__wrapped__ = wrapped
+
+    def __call__(self, image: NDArray[Any]) -> NDArray[Any]:
+        import torch
+
+        tensor = torch.as_tensor(np.ascontiguousarray(image))
+        result = self.__wrapped__(tensor)
+        return np.asarray(result.detach().cpu())
+
+    def __repr__(self) -> str:
+        return repr(self.__wrapped__)
+
+
+def build_preprocessing(steps: Sequence[PreprocessingStep]) -> _PreprocessingTransform:
+    """Build preprocessing pipeline from config.
+
+    Builds a torchvision.transforms.v2 pipeline and wraps it so the returned
+    callable accepts a **numpy CHW array** and returns a **numpy CHW array**.
+    Internally the image is converted to a torch tensor for the v2 transforms,
+    then converted back to numpy afterwards.
+
+    See: https://pytorch.org/vision/stable/transforms.html
+
+    Parameters
+    ----------
+    steps : list[PreprocessingStep]
+        List of preprocessing steps from config.
+
+    Returns
+    -------
+    Callable[[NDArray[Any]], NDArray[Any]]
+        Wrapped transform: numpy CHW in, numpy CHW out.
+    """
+    import torch
+    from torchvision.transforms import InterpolationMode, v2
+
+    # Special parameter converters for non-primitive types
+    def _resolve_dtype(name: object) -> torch.dtype:
+        attr = str(name)
+        result = getattr(torch, attr, None)
+        if result is None or not isinstance(result, torch.dtype):
+            raise ValueError(f"Unknown torch dtype: '{name}'. Example: 'float32', 'uint8'.")
+        return result
+
+    def _resolve_interpolation(name: object) -> InterpolationMode:
+        attr = str(name)
+        result = getattr(InterpolationMode, attr, None)
+        if result is None:
+            raise ValueError(f"Unknown InterpolationMode: '{name}'. Example: 'BILINEAR', 'NEAREST'.")
+        return result
+
+    param_converters = {
+        "dtype": _resolve_dtype,
+        "interpolation": _resolve_interpolation,
+    }
+
+    logger.debug("Building preprocessing pipeline: %s", [s.step for s in steps])
+
+    ops: list[Any] = []
+    for step in steps:
+        params = dict(step.params)
+
+        # Convert special parameter types
+        for key, converter in param_converters.items():
+            if key in params:
+                params[key] = converter(params[key])
+
+        # Get transform class and instantiate
+        transform_cls = getattr(v2, step.step, None)
+        if transform_cls is None:
+            raise ValueError(f"Unknown transform: '{step.step}'. Check torchvision.transforms.v2 docs.")
+        ops.append(transform_cls(**params))
+
+    composed = v2.Compose(ops)
+    return _PreprocessingTransform(composed)

dataeval_flow/runner.py

@@ -0,0 +1,118 @@
+"""Shared CLI/container runner — loads config, runs tasks, writes reports."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from dataeval_flow.config._models import PipelineConfig
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+def _resolve_config(config_arg: Path | str | None, data_dir: Path) -> PipelineConfig:
+    """Resolve and load config from an explicit path or auto-discover from data root."""
+    from dataeval_flow.config._loader import load_config, load_config_folder
+
+    if config_arg is not None:
+        config_path = Path(config_arg)
+        if not config_path.is_absolute():
+            config_path = data_dir / config_path
+    else:
+        config_path = data_dir
+
+    if config_path.is_file():
+        return load_config(config_path)
+    if config_path.is_dir():
+        return load_config_folder(config_path)
+
+    msg = f"Config path not found: {config_path}"
+    raise FileNotFoundError(msg)
+
+
+def run(
+    config_arg: Path | str | None,
+    output_dir: Path | None = None,
+    data_dir: Path | None = None,
+    verbosity: int = 0,
+    cache_dir: Path | None = None,
+) -> int:
+    """Load config, execute all tasks, and write reports.
+
+    This is the shared entry point for CLI (``__main__.py``) and container
+    (container) usage.  For programmatic use, prefer
+    :func:`~dataeval_flow.load_config` + :func:`~dataeval_flow.run_tasks`.
+
+    Parameters
+    ----------
+    config_arg : Path | str | None
+        Path to config file or folder, or None for auto-discovery at data root.
+    output_dir : Path | None
+        Directory for results, logs, and reports.  When ``None``, results
+        are printed to the console only — no file artifacts are created.
+    data_dir : Path | None
+        Root directory for data files. Defaults to ``$DATAEVAL_DATA`` or current directory.
+    verbosity : int
+        Console verbosity (0=quiet, 1=text report, 2=+INFO, 3=+DEBUG).
+    cache_dir : Path | None
+        Directory for disk-backed computation cache (embeddings, metadata, stats).
+
+    Returns
+    -------
+    int
+        0 if all tasks succeed, 1 if any fail.
+    """
+    from dataeval_flow._logging import configure_log_levels, flush_logs, setup_logging
+    from dataeval_flow.config._loader import get_data_dir
+    from dataeval_flow.workflow import run_tasks
+
+    setup_logging(output_dir, verbosity)
+
+    resolved_data = get_data_dir(data_dir)
+    config = _resolve_config(config_arg, resolved_data)
+
+    if config.logging:
+        configure_log_levels(config.logging.app_level, config.logging.lib_level)
+
+    if not config.tasks:
+        logger.info("No tasks defined in config.")
+        return 0
+
+    results = run_tasks(config, data_dir=resolved_data, cache_dir=cache_dir)
+
+    failures = 0
+    merged: dict[str, dict] = {}
+    text_parts: list[str] = []
+
+    for task, result in zip(config.tasks, results, strict=True):
+        if not result.success:
+            logger.error("  FAILED: %s", task.name)
+            for error in result.errors:
+                logger.error("    %s", error)
+            failures += 1
+            flush_logs()
+            continue
+
+        # --- Text report: summary (no flag) or full detail (-v) ---
+        print(result.report(detailed=verbosity >= 1))
+
+        # --- Collect for file output ---
+        merged[task.name] = result.to_dict()
+        text_parts.append(result.report(detailed=True))
+
+        logger.info("  OK: %s", task.name)
+        flush_logs()
+
+    # --- Write file artifacts (only when output_dir is set) ---
+    if output_dir is not None and merged:
+        import json as json_mod
+
+        output_dir.mkdir(parents=True, exist_ok=True)
+        (output_dir / "result.json").write_text(json_mod.dumps(merged, indent=2), encoding="utf-8")
+        (output_dir / "result.txt").write_text("\n".join(text_parts), encoding="utf-8")
+        logger.info("  Wrote result.json and result.txt to %s", output_dir)
+
+    logger.info("Done. %d/%d succeeded.", len(config.tasks) - failures, len(config.tasks))
+    return 1 if failures else 0

dataeval_flow/selection.py

@@ -0,0 +1,50 @@
+"""Selection convenience builder wrapping DataEval."""
+
+__all__ = ["build_selection"]
+
+from typing import TYPE_CHECKING, TypeVar
+
+import dataeval.selection as sel
+from dataeval.protocols import AnnotatedDataset
+from dataeval.selection import Select
+
+if TYPE_CHECKING:
+    from dataeval_flow.config.schemas import SelectionStep
+
+T = TypeVar("T")
+
+
+def build_selection(dataset: AnnotatedDataset[T], steps: list["SelectionStep"]) -> Select[T]:
+    """Build selection pipeline from config.
+
+    Pass-through to dataeval.selection - no custom logic.
+
+    Parameters
+    ----------
+    dataset : MaiteDataset
+        Input dataset to wrap with selections.
+    steps : list[SelectionStep]
+        Selection steps from config.
+
+    Returns
+    -------
+    Select
+        Dataset wrapped with selection criteria.
+
+    Example
+    -------
+    >>> from dataeval_flow.config import SelectionStep
+    >>> steps = [
+    ...     SelectionStep(type="Limit", params={"size": 10000}),
+    ...     SelectionStep(type="ClassFilter", params={"classes": [0, 1, 2]}),
+    ... ]
+    >>> filtered = build_selection(dataset, steps)
+    """
+    selections = []
+    for step in steps:
+        selection_cls = getattr(sel, step.type, None)
+        if selection_cls is None:
+            raise ValueError(f"Unknown selection type: '{step.type}'. Check dataeval.selection docs.")
+        selections.append(selection_cls(**step.params))
+
+    return Select(dataset, selections=selections)