dataeval-flow 0.1.0__py3-none-any.whl

dataeval_flow/workflows/prioritization/__init__.py

@@ -0,0 +1 @@
+"""Data prioritization workflow."""

dataeval_flow/workflows/prioritization/outputs.py

@@ -0,0 +1,122 @@
+"""Data prioritization workflow outputs."""
+
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias
+
+from pydantic import BaseModel, Field
+from typing_extensions import TypedDict, TypeIs
+
+from dataeval_flow.config import ResultMetadata
+from dataeval_flow.workflow.base import Reportable, WorkflowOutputsBase, WorkflowReportBase
+
+if TYPE_CHECKING:
+    from dataeval_flow.workflow import WorkflowResult
+
+__all__ = [
+    "CleaningSummaryDict",
+    "DataPrioritizationMetadata",
+    "DataPrioritizationOutputs",
+    "DataPrioritizationRawOutputs",
+    "DataPrioritizationReport",
+    "DataPrioritizationResult",
+    "PerDatasetPrioritizationDict",
+    "is_prioritization_result",
+]
+
+
+# ---------------------------------------------------------------------------
+# TypedDicts for serialized outputs
+# ---------------------------------------------------------------------------
+
+
+class PerDatasetPrioritizationDict(TypedDict):
+    """Prioritization results for a single additional dataset."""
+
+    source_name: str
+    original_size: int
+    cleaned_size: int
+    prioritized_indices: list[int]
+    scores: list[float] | None
+
+
+class CleaningSummaryDict(TypedDict):
+    """Summary of the cleaning step (combined across all datasets)."""
+
+    total_combined: int
+    outliers_flagged: int
+    duplicates_flagged: int
+    total_removed: int
+
+
+# ---------------------------------------------------------------------------
+# Pydantic output models
+# ---------------------------------------------------------------------------
+
+
+class DataPrioritizationRawOutputs(WorkflowOutputsBase):
+    """Machine-readable results from the data-prioritization workflow."""
+
+    reference_size: int = Field(
+        default=0,
+        description="Number of items in the reference dataset.",
+    )
+    method: str = Field(
+        default="",
+        description="Ranking method used.",
+    )
+    order: str = Field(
+        default="",
+        description="Sort direction used.",
+    )
+    policy: str = Field(
+        default="",
+        description="Selection policy used.",
+    )
+    cleaning_summary: CleaningSummaryDict | None = Field(
+        default=None,
+        description="Cleaning step summary. None if cleaning was skipped.",
+    )
+    prioritizations: list[PerDatasetPrioritizationDict] = Field(
+        default_factory=list,
+        description="Per-dataset prioritization results.",
+    )
+
+
+class DataPrioritizationReport(WorkflowReportBase):
+    """Human-readable report for the data-prioritization workflow."""
+
+    findings: list[Reportable] = Field(default_factory=list)
+
+
+class DataPrioritizationOutputs(BaseModel):
+    """Complete data-prioritization workflow output."""
+
+    raw: DataPrioritizationRawOutputs
+    report: DataPrioritizationReport
+
+
+class DataPrioritizationMetadata(ResultMetadata):
+    """Metadata for the data-prioritization workflow."""
+
+    mode: Literal["advisory", "preparatory"] = "advisory"
+    method: str = ""
+    order: str = ""
+    policy: str = ""
+    cleaning_enabled: bool = False
+    items_removed_by_cleaning: int = 0
+    per_source_clean_indices: dict[str, list[int]] = Field(default_factory=dict)
+    per_source_prioritized_indices: dict[str, list[int]] = Field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# Type alias and TypeIs guard for type narrowing
+# ---------------------------------------------------------------------------
+
+#: Fully typed result alias for the data-prioritization workflow.
+DataPrioritizationResult: TypeAlias = "WorkflowResult[DataPrioritizationMetadata, DataPrioritizationOutputs]"
+
+
+def is_prioritization_result(
+    result: "WorkflowResult[Any, Any]",
+) -> TypeIs["WorkflowResult[DataPrioritizationMetadata, DataPrioritizationOutputs]"]:
+    """Narrow a generic ``WorkflowResult`` to a data-prioritization result."""
+    return isinstance(result.metadata, DataPrioritizationMetadata)

dataeval_flow/workflows/prioritization/params.py

@@ -0,0 +1,124 @@
+"""Data prioritization workflow parameters."""
+
+from collections.abc import Sequence
+from typing import ClassVar, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from dataeval_flow.workflow.base import WorkflowParametersBase
+
+__all__ = [
+    "CleaningConfig",
+    "DataPrioritizationHealthThresholds",
+    "DataPrioritizationParameters",
+]
+
+MethodType = Literal["knn", "kmeans_distance", "kmeans_complexity", "hdbscan_distance", "hdbscan_complexity"]
+OrderType = Literal["easy_first", "hard_first"]
+PolicyType = Literal["difficulty", "stratified", "class_balanced"]
+
+
+class CleaningConfig(BaseModel):
+    """Optional cleaning sub-config for outlier/duplicate removal before prioritization.
+
+    When provided, outlier and duplicate detection runs across all datasets
+    before prioritization.  Flagged items are excluded from the ranking.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    outlier_method: Literal["adaptive", "zscore", "modzscore", "iqr"] = Field(
+        description="Statistical method for outlier detection.",
+    )
+    outlier_flags: Sequence[Literal["dimension", "pixel", "visual"]] = Field(
+        min_length=1,
+        description="Image statistics groups for outlier detection. At least one required.",
+    )
+    outlier_threshold: float | None = Field(
+        default=None,
+        ge=0.0,
+        description="Custom threshold (None = use DataEval default for chosen method).",
+    )
+    duplicate_flags: Sequence[Literal["hash_basic", "hash_d4"]] | None = Field(
+        default=None,
+        description="Hash flag groups for duplicate detection. None = DataEval default (hash_basic).",
+    )
+    duplicate_merge_near: bool = Field(
+        default=True,
+        description="Merge overlapping near-duplicate groups from different detection methods.",
+    )
+    duplicate_exact_only: bool = Field(
+        default=False,
+        description="When True, only flag exact duplicates — skip near-duplicate detection.",
+    )
+
+
+class DataPrioritizationHealthThresholds(BaseModel):
+    """Thresholds that control finding severity for the prioritization report."""
+
+    cleaning_removed_pct_warning: float = Field(
+        default=20.0,
+        ge=0.0,
+        le=100.0,
+        description="Warn if cleaning removes more than this percentage of the combined dataset.",
+    )
+
+
+class DataPrioritizationParameters(WorkflowParametersBase):
+    """Parameters for the data-prioritization workflow.
+
+    Requires at least two sources: the first is the reference (labeled) dataset,
+    and subsequent sources are unlabeled data pools to prioritize.
+    """
+
+    # --- Prioritization method ---
+    method: MethodType = Field(
+        default="knn",
+        description="Ranking method: knn, kmeans_distance, kmeans_complexity, hdbscan_distance, hdbscan_complexity.",
+    )
+    k: int | None = Field(
+        default=None,
+        gt=0,
+        description="Number of nearest neighbors for knn method. None = sqrt(n_samples).",
+    )
+    c: int | None = Field(
+        default=None,
+        gt=0,
+        description="Number of clusters for clustering methods. None = sqrt(n_samples).",
+    )
+    n_init: int | Literal["auto"] = Field(
+        default="auto",
+        description="Number of K-means initializations (kmeans methods only).",
+    )
+    max_cluster_size: int | None = Field(
+        default=None,
+        gt=0,
+        description="Maximum cluster size for HDBSCAN methods.",
+    )
+
+    # --- Order and policy ---
+    order: OrderType = Field(
+        default="hard_first",
+        description="Sort direction: easy_first (prototypical first) or hard_first (novel/challenging first).",
+    )
+    policy: PolicyType = Field(
+        default="difficulty",
+        description="Selection policy: difficulty (direct ordering), stratified (binned), class_balanced.",
+    )
+    num_bins: int = Field(
+        default=50,
+        gt=0,
+        description="Number of bins for stratified policy.",
+    )
+
+    # --- Optional cleaning ---
+    cleaning: CleaningConfig | None = Field(
+        default=None,
+        description="Optional cleaning config. When set, outlier/duplicate detection runs before prioritization.",
+    )
+
+    # --- Health thresholds ---
+    health_thresholds: DataPrioritizationHealthThresholds = Field(
+        default_factory=DataPrioritizationHealthThresholds,
+        description="Warning thresholds for the prioritization report.",
+    )

dataeval_flow/workflows/prioritization/report.py

@@ -0,0 +1,117 @@
+"""Findings builders for the data-prioritization workflow."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from dataeval_flow.workflow.base import Reportable
+from dataeval_flow.workflows.prioritization.outputs import (
+    CleaningSummaryDict,
+    DataPrioritizationRawOutputs,
+    PerDatasetPrioritizationDict,
+)
+from dataeval_flow.workflows.prioritization.params import (
+    DataPrioritizationHealthThresholds,
+    DataPrioritizationParameters,
+)
+
+
+def _severity_for_cleaning(
+    removed_pct: float,
+    thresholds: DataPrioritizationHealthThresholds,
+) -> Literal["ok", "info", "warning"]:
+    """Determine severity based on percentage of items removed by cleaning."""
+    if removed_pct >= thresholds.cleaning_removed_pct_warning:
+        return "warning"
+    if removed_pct > 0:
+        return "info"
+    return "ok"
+
+
+def _build_cleaning_finding(
+    summary: CleaningSummaryDict,
+    thresholds: DataPrioritizationHealthThresholds,
+) -> Reportable:
+    """Build a finding for the cleaning step."""
+    total = summary["total_combined"]
+    removed = summary["total_removed"]
+    removed_pct = (removed / total * 100) if total > 0 else 0.0
+    severity = _severity_for_cleaning(removed_pct, thresholds)
+
+    data: dict[str, Any] = {
+        "brief": f"{removed} items ({removed_pct:.1f}%)",
+        "total_combined": total,
+        "outliers_flagged": summary["outliers_flagged"],
+        "duplicates_flagged": summary["duplicates_flagged"],
+        "total_removed": removed,
+        "removed_pct": f"{removed_pct:.1f}%",
+    }
+
+    description = (
+        f"Pruning removed {removed}/{total} items ({removed_pct:.1f}%): "
+        f"{summary['outliers_flagged']} outliers, {summary['duplicates_flagged']} duplicates"
+    )
+
+    return Reportable(
+        report_type="key_value",
+        severity=severity,
+        title="Pruning",
+        data=data,
+        description=description,
+    )
+
+
+def _build_prioritization_finding(
+    result: PerDatasetPrioritizationDict,
+    method: str,
+    order: str,
+    policy: str,
+) -> Reportable:
+    """Build a finding for a single dataset's prioritization results."""
+    n_items = result["cleaned_size"]
+    top_n = min(10, len(result["prioritized_indices"]))
+    top_indices = result["prioritized_indices"][:top_n]
+
+    data: dict[str, Any] = {
+        "brief": f"{n_items} items",
+        "source": result["source_name"],
+        "original_size": result["original_size"],
+        "cleaned_size": result["cleaned_size"],
+        "method": method,
+        "order": order,
+        "policy": policy,
+        "top_indices": top_indices,
+    }
+
+    if result["scores"] is not None:
+        top_scores = result["scores"][:top_n]
+        data["top_scores"] = [round(s, 4) for s in top_scores]
+
+    description = f"{result['source_name']}: {n_items} items prioritized via {method} ({order}, {policy})"
+
+    return Reportable(
+        report_type="key_value",
+        severity="info",
+        title=f"Prioritization: {result['source_name']}",
+        data=data,
+        description=description,
+    )
+
+
+def build_findings(
+    raw: DataPrioritizationRawOutputs,
+    params: DataPrioritizationParameters,
+) -> list[Reportable]:
+    """Build all report findings from raw results."""
+    findings: list[Reportable] = []
+
+    # Cleaning finding
+    if raw.cleaning_summary is not None:
+        findings.append(_build_cleaning_finding(raw.cleaning_summary, params.health_thresholds))
+
+    # Per-dataset prioritization findings
+    findings.extend(
+        _build_prioritization_finding(result, raw.method, raw.order, raw.policy) for result in raw.prioritizations
+    )
+
+    return findings