dataeval-flow 0.1.0__py3-none-any.whl

dataeval_flow/workflows/splitting/workflow.py

@@ -0,0 +1,371 @@
+"""Dataset splitting workflow implementation."""
+
+import logging
+import traceback
+from typing import Any
+
+import numpy as np
+from pydantic import BaseModel
+
+from dataeval_flow.workflow import WorkflowContext, WorkflowResult
+from dataeval_flow.workflows.splitting.outputs import (
+    DataSplittingMetadata,
+    DataSplittingOutputs,
+    DataSplittingRawOutputs,
+    DataSplittingReport,
+    SplitInfo,
+)
+from dataeval_flow.workflows.splitting.params import DataSplittingParameters
+from dataeval_flow.workflows.splitting.report import build_findings
+
+__all__ = ["DataSplittingWorkflow"]
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Serialization helpers
+# ---------------------------------------------------------------------------
+
+
+def _serialize_label_stats(stats: Any) -> dict[str, Any]:
+    """Convert a LabelStatsResult to a plain dict."""
+    if stats is None:
+        return {}
+    result: dict[str, Any] = {}
+    for key in (
+        "label_counts_per_class",
+        "image_counts_per_class",
+        "class_count",
+        "label_count",
+        "image_count",
+        "index2label",
+    ):
+        val = stats.get(key, None) if hasattr(stats, "get") else getattr(stats, key, None)
+        if val is not None:
+            if hasattr(val, "tolist"):
+                val = val.tolist()
+            result[key] = val
+    return result
+
+
+def _serialize_balance(output: Any) -> dict[str, Any]:
+    """Convert BalanceOutput to a plain dict."""
+    result: dict[str, Any] = {}
+    for attr in ("balance", "factors", "classwise"):
+        df = getattr(output, attr, None)
+        if df is not None:
+            result[attr] = df.to_dicts() if hasattr(df, "to_dicts") else str(df)
+    return result
+
+
+def _serialize_diversity(output: Any) -> dict[str, Any]:
+    """Convert DiversityOutput to a plain dict."""
+    result: dict[str, Any] = {}
+    for attr in ("factors", "classwise"):
+        df = getattr(output, attr, None)
+        if df is not None:
+            result[attr] = df.to_dicts() if hasattr(df, "to_dicts") else str(df)
+    return result
+
+
+def _serialize_coverage(coverage_result: Any) -> dict[str, Any]:
+    """Convert CoverageResult to a plain dict."""
+    result: dict[str, Any] = {}
+    for key in ("uncovered_indices", "critical_value_radii", "coverage_radius"):
+        val = coverage_result.get(key, None) if hasattr(coverage_result, "get") else getattr(coverage_result, key, None)
+        if val is not None:
+            if hasattr(val, "tolist"):
+                val = val.tolist()
+            result[key] = val
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Embedding-space coverage helper (step 7)
+# Extracted from _execute to satisfy C901 complexity limit.
+# "Coverage" here refers to dataeval.core.coverage_adaptive(), not test coverage.
+# ---------------------------------------------------------------------------
+
+
+def _run_coverage(
+    ds_ctx: Any,
+    dataset: Any,
+    fold_infos: list[SplitInfo],
+    test_indices: list[int],
+    params: DataSplittingParameters,
+) -> dict[str, Any] | None:
+    """Run per-split coverage assessment if model is provided."""
+    if ds_ctx.extractor is None:
+        logger.info("Step 7: Skipping coverage (no model provided)")
+        return None
+
+    from dataeval.core import coverage_adaptive
+
+    from dataeval_flow.embeddings import build_embeddings
+
+    logger.info("Step 7: Per-split coverage assessment")
+    embeddings_obj = build_embeddings(
+        dataset,
+        ds_ctx.extractor,
+        transforms=ds_ctx.transforms,
+        batch_size=ds_ctx.batch_size,
+    )
+    all_embeddings = np.array(embeddings_obj)
+
+    # Normalize to [0, 1]
+    emb_min = all_embeddings.min(axis=0, keepdims=True)
+    emb_max = all_embeddings.max(axis=0, keepdims=True)
+    emb_range = emb_max - emb_min
+    emb_range[emb_range == 0] = 1.0
+    all_embeddings = (all_embeddings - emb_min) / emb_range
+
+    for fold_info in fold_infos:
+        train_embs = all_embeddings[fold_info.train_indices]
+        val_embs = all_embeddings[fold_info.val_indices]
+        fold_info.coverage_train = _serialize_coverage(
+            coverage_adaptive(train_embs, params.num_observations, params.coverage_percent)
+        )
+        fold_info.coverage_val = _serialize_coverage(
+            coverage_adaptive(val_embs, params.num_observations, params.coverage_percent)
+        )
+
+    coverage_test_data: dict[str, Any] | None = None
+    if test_indices:
+        test_embs = all_embeddings[test_indices]
+        coverage_test_data = _serialize_coverage(
+            coverage_adaptive(test_embs, params.num_observations, params.coverage_percent)
+        )
+    return coverage_test_data
+
+
+# ---------------------------------------------------------------------------
+# Workflow class
+# ---------------------------------------------------------------------------
+
+
+class DataSplittingWorkflow:
+    """Dataset splitting workflow.
+
+    Assesses dataset balance/diversity, produces stratified train/val/test
+    splits, and optionally rebalances the train split.
+    """
+
+    @property
+    def name(self) -> str:
+        """Workflow identifier."""
+        return "data-splitting"
+
+    @property
+    def description(self) -> str:
+        """Human-readable description."""
+        return (
+            "Assess dataset balance/diversity, produce stratified train/val/test "
+            "splits, and optionally rebalance the train split."
+        )
+
+    @property
+    def params_schema(self) -> type[BaseModel]:
+        """Pydantic model for workflow parameters."""
+        return DataSplittingParameters
+
+    @property
+    def output_schema(self) -> type[BaseModel]:
+        """Pydantic model for workflow output."""
+        return DataSplittingOutputs
+
+    def execute(
+        self,
+        context: WorkflowContext,
+        params: BaseModel | None = None,
+    ) -> WorkflowResult[DataSplittingMetadata, DataSplittingOutputs]:
+        """Execute the splitting workflow."""
+        if not isinstance(context, WorkflowContext):
+            msg = f"Expected WorkflowContext, got {type(context).__name__}"
+            raise TypeError(msg)
+
+        if params is not None and not isinstance(params, DataSplittingParameters):
+            msg = f"Expected DataSplittingParameters, got {type(params).__name__}"
+            raise TypeError(msg)
+
+        p = params or DataSplittingParameters()
+
+        try:
+            return self._execute(context, p)
+        except Exception:
+            logger.exception("Splitting workflow failed")
+            return WorkflowResult(
+                name=self.name,
+                success=False,
+                data=_empty_outputs(),
+                errors=[traceback.format_exc()],
+                metadata=DataSplittingMetadata(),
+            )
+
+    def _execute(
+        self,
+        context: WorkflowContext,
+        params: DataSplittingParameters,
+    ) -> WorkflowResult[DataSplittingMetadata, DataSplittingOutputs]:
+        from dataeval.bias import Balance, Diversity
+        from dataeval.core import label_stats
+        from dataeval.utils.data import split_dataset
+
+        from dataeval_flow.metadata import build_metadata
+        from dataeval_flow.selection import build_selection
+
+        # --- Resolve single dataset ---
+        if not context.dataset_contexts:
+            msg = "No datasets provided"
+            raise ValueError(msg)
+
+        ds_name = next(iter(context.dataset_contexts))
+        ds_ctx = context.dataset_contexts[ds_name]
+        dataset: Any = ds_ctx.dataset
+
+        # Apply selection if configured
+        if ds_ctx.selection_steps:
+            dataset = build_selection(dataset, list(ds_ctx.selection_steps))
+
+        dataset_size = len(dataset)
+        logger.info("Step 1: Building metadata for %s (%d items)", ds_name, dataset_size)
+
+        # --- Step 1: Build Metadata ---
+        metadata = build_metadata(dataset)
+
+        # --- Step 2: Pre-split bias assessment ---
+        logger.info("Step 2: Pre-split bias assessment")
+        balance_output = Balance().evaluate(metadata)
+        diversity_output = Diversity().evaluate(metadata)
+
+        pre_split_balance = _serialize_balance(balance_output)
+        pre_split_diversity = _serialize_diversity(diversity_output)
+
+        # --- Step 3: Full-dataset label stats ---
+        logger.info("Step 3: Label statistics (full dataset)")
+        class_labels = metadata.class_labels
+        index2label = metadata.index2label if hasattr(metadata, "index2label") else None
+        full_stats = label_stats(class_labels, index2label=index2label)
+        label_stats_full = _serialize_label_stats(full_stats)
+
+        # --- Step 4: Split ---
+        logger.info(
+            "Step 4: Splitting dataset (num_folds=%d, stratify=%s, test_frac=%s, val_frac=%s)",
+            params.num_folds,
+            params.stratify,
+            params.test_frac,
+            params.val_frac,
+        )
+        # Use metadata if split_on is specified, otherwise use dataset directly
+        split_input: Any = metadata if params.split_on else dataset
+        # val_frac is only valid for single-fold; multi-fold uses 1/num_folds automatically
+        val_frac = params.val_frac if params.num_folds == 1 else 0.0
+        splits = split_dataset(
+            split_input,
+            num_folds=params.num_folds,
+            stratify=params.stratify,
+            split_on=params.split_on,
+            test_frac=params.test_frac,
+            val_frac=val_frac,
+        )
+
+        test_indices = splits.test.tolist()
+
+        # --- Step 5: Optional rebalancing ---
+        # ClassBalance is a selection step that operates on a Select-wrapped dataset.
+        # For now we store the raw indices; rebalancing modifies train indices.
+        fold_infos: list[SplitInfo] = []
+        for i, fold in enumerate(splits.folds):
+            train_idx = fold.train.tolist()
+            val_idx = fold.val.tolist()
+
+            if params.rebalance_method is not None:
+                logger.info("Step 5: Rebalancing fold %d train split (method=%s)", i, params.rebalance_method)
+                from dataeval.selection import ClassBalance, Indices, Select
+
+                train_selected = Select(dataset, Indices(train_idx))
+                ClassBalance(method=params.rebalance_method)(train_selected)
+                train_idx = train_selected.resolve_indices()
+
+            fold_infos.append(
+                SplitInfo(
+                    fold=i,
+                    train_indices=train_idx,
+                    val_indices=val_idx,
+                )
+            )
+
+        # --- Step 6: Per-split label stats ---
+        logger.info("Step 6: Per-split label statistics")
+        for fold_info in fold_infos:
+            train_labels = class_labels[fold_info.train_indices]
+            val_labels = class_labels[fold_info.val_indices]
+            fold_info.label_stats_train = _serialize_label_stats(label_stats(train_labels, index2label=index2label))
+            fold_info.label_stats_val = _serialize_label_stats(label_stats(val_labels, index2label=index2label))
+
+        test_labels = class_labels[test_indices] if test_indices else np.array([], dtype=np.intp)
+        label_stats_test = (
+            _serialize_label_stats(label_stats(test_labels, index2label=index2label)) if len(test_labels) > 0 else {}
+        )
+
+        # --- Step 7: Per-split coverage (if model provided) ---
+        coverage_test_data = _run_coverage(
+            ds_ctx,
+            dataset,
+            fold_infos,
+            test_indices,
+            params,
+        )
+
+        # --- Build raw outputs ---
+        raw = DataSplittingRawOutputs(
+            dataset_size=dataset_size,
+            pre_split_balance=pre_split_balance,
+            pre_split_diversity=pre_split_diversity,
+            label_stats_full=label_stats_full,
+            test_indices=test_indices,
+            label_stats_test=label_stats_test,
+            coverage_test=coverage_test_data,
+            folds=fold_infos,
+        )
+
+        # --- Build findings ---
+        findings = build_findings(raw)
+
+        # --- Build split sizes for metadata ---
+        fold0 = fold_infos[0] if fold_infos else None
+        split_sizes = {
+            "train": len(fold0.train_indices) if fold0 else 0,
+            "val": len(fold0.val_indices) if fold0 else 0,
+            "test": len(test_indices),
+        }
+
+        report = DataSplittingReport(
+            summary=f"Dataset splitting: {dataset_size} items → {len(fold_infos)} fold(s)",
+            findings=findings,
+        )
+
+        outputs = DataSplittingOutputs(raw=raw, report=report)
+
+        metadata = DataSplittingMetadata(
+            num_folds=params.num_folds,
+            stratified=params.stratify,
+            split_on=params.split_on,
+            rebalance_method=params.rebalance_method,
+            split_sizes=split_sizes,
+        )
+
+        return WorkflowResult(
+            name=self.name,
+            success=True,
+            data=outputs,
+            metadata=metadata,
+        )
+
+
+def _empty_outputs() -> DataSplittingOutputs:
+    """Create empty outputs for error cases."""
+    return DataSplittingOutputs(
+        raw=DataSplittingRawOutputs(dataset_size=0),
+        report=DataSplittingReport(summary="Splitting workflow failed"),
+    )

dataeval_flow-0.1.0.dist-info/METADATA

@@ -0,0 +1,305 @@
+Metadata-Version: 2.4
+Name: dataeval-flow
+Version: 0.1.0
+Summary: DataEval Workflows container for data evaluation
+Project-URL: Repository, https://gitlab.jatic.net/jatic/aria/dataeval-flow
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Classifier: Typing :: Typed
+Requires-Python: <3.14,>=3.10
+Requires-Dist: dataeval==1.0.4
+Requires-Dist: datasets>=4.0.0
+Requires-Dist: maite-datasets>=0.0.12
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: all-cpu
+Requires-Dist: onnx>=1.15; extra == 'all-cpu'
+Requires-Dist: onnxruntime>=1.20; extra == 'all-cpu'
+Requires-Dist: opencv-python-headless>=4.8.0; extra == 'all-cpu'
+Requires-Dist: textual>=3.0; extra == 'all-cpu'
+Requires-Dist: torch>=2.2.0; extra == 'all-cpu'
+Requires-Dist: torchvision>=0.17.0; extra == 'all-cpu'
+Provides-Extra: all-cu118
+Requires-Dist: onnx>=1.15; extra == 'all-cu118'
+Requires-Dist: onnxruntime-gpu>=1.20; extra == 'all-cu118'
+Requires-Dist: opencv-python-headless>=4.8.0; extra == 'all-cu118'
+Requires-Dist: textual>=3.0; extra == 'all-cu118'
+Requires-Dist: torch>=2.2.0; extra == 'all-cu118'
+Requires-Dist: torchvision>=0.17.0; extra == 'all-cu118'
+Provides-Extra: all-cu124
+Requires-Dist: onnx>=1.15; extra == 'all-cu124'
+Requires-Dist: onnxruntime-gpu>=1.20; extra == 'all-cu124'
+Requires-Dist: opencv-python-headless>=4.8.0; extra == 'all-cu124'
+Requires-Dist: textual>=3.0; extra == 'all-cu124'
+Requires-Dist: torch>=2.2.0; extra == 'all-cu124'
+Requires-Dist: torchvision>=0.17.0; extra == 'all-cu124'
+Provides-Extra: all-cu128
+Requires-Dist: onnx>=1.15; extra == 'all-cu128'
+Requires-Dist: onnxruntime-gpu>=1.23.2; extra == 'all-cu128'
+Requires-Dist: opencv-python-headless>=4.8.0; extra == 'all-cu128'
+Requires-Dist: textual>=3.0; extra == 'all-cu128'
+Requires-Dist: torch>=2.2.0; extra == 'all-cu128'
+Requires-Dist: torchvision>=0.17.0; extra == 'all-cu128'
+Provides-Extra: app
+Requires-Dist: textual>=3.0; extra == 'app'
+Provides-Extra: cpu
+Requires-Dist: torch>=2.2.0; extra == 'cpu'
+Requires-Dist: torchvision>=0.17.0; extra == 'cpu'
+Provides-Extra: cu118
+Requires-Dist: torch>=2.2.0; extra == 'cu118'
+Requires-Dist: torchvision>=0.17.0; extra == 'cu118'
+Provides-Extra: cu124
+Requires-Dist: torch>=2.2.0; extra == 'cu124'
+Requires-Dist: torchvision>=0.17.0; extra == 'cu124'
+Provides-Extra: cu128
+Requires-Dist: torch>=2.2.0; extra == 'cu128'
+Requires-Dist: torchvision>=0.17.0; extra == 'cu128'
+Provides-Extra: onnx
+Requires-Dist: onnx>=1.15; extra == 'onnx'
+Requires-Dist: onnxruntime>=1.20; extra == 'onnx'
+Provides-Extra: onnx-gpu
+Requires-Dist: onnx>=1.15; extra == 'onnx-gpu'
+Requires-Dist: onnxruntime-gpu>=1.20; extra == 'onnx-gpu'
+Provides-Extra: opencv
+Requires-Dist: opencv-python-headless>=4.8.0; extra == 'opencv'
+Description-Content-Type: text/markdown
+
+# DataEval Workflows
+
+Workflow orchestration for DataEval with GPU support.
+
+## Quick Start
+
+```bash
+# 1. Build CUDA 11.8 container
+docker build -f docker/Dockerfile.cu118 -t dataeval:cu118 .
+
+# 2. Show help
+docker run dataeval:cu118
+
+# 3. Run with data and output
+docker run --gpus all \
+  --mount type=bind,source=/path/to/data,target=/dataeval,readonly \
+  --mount type=bind,source=/path/to/output,target=/output \
+  dataeval:cu118
+```
+
+## Requirements
+
+| Requirement | Version |
+|-------------|---------|
+| Docker | >= 20.10 |
+| NVIDIA GPU | Any (for GPU mode) |
+| NVIDIA Driver | >= 520 (for GPU mode) |
+| CUDA | 11.8.0 (for GPU mode) |
+
+### Verify GPU Access
+
+```bash
+docker run --rm --gpus all nvidia/cuda:11.8.0-base-ubuntu22.04 nvidia-smi
+```
+
+## Volume Mounts
+
+| Path | Mode | Purpose |
+|------|------|---------|
+| `/dataeval` | ro | Data directory — datasets, models, configs (required) |
+| `/output` | rw | Results (required) |
+| `/cache` | rw | Computation cache (optional) |
+
+### File Permissions
+
+The container runs as a non-root user (`dataeval`, UID 1000). Mounted directories for `/output` and `/cache` must be writable by the container process. There are two approaches:
+
+#### Option 1: Pass your host UID (recommended)
+
+Use `--user` to run the container as your host user, so mounted directories are naturally writable:
+
+```bash
+docker run --gpus all \
+  --user "$(id -u):$(id -g)" \
+  --mount type=bind,source=/path/to/data,target=/dataeval,readonly \
+  --mount type=bind,source=/path/to/output,target=/output \
+  dataeval:cu118
+```
+
+#### Option 2: Open directory permissions
+
+Make the output and cache directories world-writable on the host:
+
+```bash
+chmod 777 /path/to/output /path/to/cache
+```
+
+Then run without `--user`. This is simpler but less secure.
+
+### Custom Data Root
+
+The data root path can be overridden via the `DATAEVAL_DATA` environment variable:
+
+```bash
+docker run --gpus all \
+  -e DATAEVAL_DATA=/data \
+  --mount type=bind,source=/path/to/data,target=/data,readonly \
+  --mount type=bind,source=/path/to/output,target=/output \
+  dataeval:cu118
+```
+
+## Configuration
+
+Config files (YAML or JSON) can be placed anywhere in your data directory. By default, all YAML/JSON files at the root of the data mount are auto-discovered and merged.
+
+To specify a config path explicitly:
+
+```bash
+# Config folder within data directory
+docker run --gpus all \
+  --mount type=bind,source=/path/to/data,target=/dataeval,readonly \
+  --mount type=bind,source=/path/to/output,target=/output \
+  dataeval:cu118 --config config/
+
+# Single config file
+docker run --gpus all \
+  --mount type=bind,source=/path/to/data,target=/dataeval,readonly \
+  --mount type=bind,source=/path/to/output,target=/output \
+  dataeval:cu118 --config params.yaml
+```
+
+Dataset and model paths in config files are resolved relative to the data root (`/dataeval` by default).
+
+## Dataset Formats
+
+Currently supported dataset structures:
+
+| Format | Structure | Example |
+|--------|-----------|---------|
+| **Dataset** | Single split, used directly | `cifar10_test/` |
+| **DatasetDict** | Multiple splits (dict), configured via config YAML | `cifar10_full/` |
+
+## CPU Fallback
+
+For machines without NVIDIA GPU:
+
+```bash
+docker build -f docker/Dockerfile.cpu -t dataeval:cpu .
+docker run dataeval:cpu  # Shows help
+docker run \
+  --mount type=bind,source=/path/to/data,target=/dataeval,readonly \
+  --mount type=bind,source=/path/to/output,target=/output \
+  dataeval:cpu
+```
+
+## CLI Modes
+
+DataEval Flow has three modes:
+
+| Command                | Purpose                                                          |
+| ---------------------- | ---------------------------------------------------------------- |
+| `dataeval-flow [opts]` | Headless execution — for automation and CI/CD pipelines          |
+| `dataeval-flow app`    | Interactive TUI dashboard — configure, execute, and view results |
+| `dataeval-flow config` | Simple CLI config builder — create/edit configs without the TUI  |
+
+### Interactive TUI (`app`)
+
+**Installation:**
+
+```bash
+uv sync --extra app          # or: pip install dataeval-flow[app]
+```
+
+**Usage:**
+
+```bash
+# Launch with a blank config
+python -m dataeval_flow app
+
+# Load an existing config for editing
+python -m dataeval_flow app --config /path/to/params.yaml
+```
+
+The TUI provides a three-pane dashboard for config editing, task execution, and result viewing. It auto-discovers available torchvision transforms, dataeval selection classes, and workflow types, generating dynamic parameter forms from their schemas.
+
+### Simple CLI Config Builder (`config`)
+
+For environments without the TUI dependency:
+
+```bash
+python -m dataeval_flow config
+python -m dataeval_flow config --config /path/to/params.yaml
+```
+
+Configs can be saved as YAML or JSON.
+
+## Dependencies
+
+- `dataeval` - Core evaluation library
+- `datasets` - Huggingface library
+- `maite-datasets` - MAITE protocol adapter
+- `maite` - MAITE protocol library
+- `pydantic` - Structural typing and schema validation
+
+## Troubleshooting
+
+### Build appears stuck at `uv sync`
+
+The Docker build may appear frozen during the `uv sync` step:
+
+```
+=> [builder 7/7] RUN uv sync --frozen --no-dev --no-install-project    1139.3s
+```
+
+**This is normal.** The step downloads ~2GB of dependencies (PyTorch, scipy, etc.) with no progress indicator.
+
+| Network Speed | Expected Build Time |
+|---------------|---------------------|
+| 100 Mbps | ~10 minutes |
+| 30 Mbps | ~20 minutes |
+| 10 Mbps | ~45 minutes |
+
+**Tip:** First build is slow; subsequent builds use Docker cache and complete in seconds.
+
+## Running Without Container
+
+The `dataeval_flow` package can be used standalone without Docker.
+
+**Installation:**
+```bash
+git clone https://gitlab.jatic.net/jatic/aria/dataeval-flow.git
+cd dataeval-flow
+uv sync
+```
+
+**CLI Usage:**
+```bash
+python -m dataeval_flow --config /path/to/config --output /path/to/output
+python -m dataeval_flow --data /path/to/data --output /path/to/output
+```
+
+**Python API Usage:**
+```python
+from pathlib import Path
+from dataeval_flow import load_config, run_tasks
+
+config = load_config(Path("/path/to/data/config.yaml"))
+results = run_tasks(config, data_dir=Path("/path/to/data"))
+print(results[0].report())
+```
+
+**Development:**
+```bash
+uv sync --group dev
+nox
+```
+
+## License
+
+MIT