dataeval-flow 0.1.0__py3-none-any.whl

dataeval_flow/workflows/drift/__init__.py

@@ -0,0 +1 @@
+"""Drift monitoring workflow."""

dataeval_flow/workflows/drift/outputs.py

@@ -0,0 +1,144 @@
+"""Drift monitoring workflow outputs."""
+
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias
+
+from pydantic import BaseModel, Field
+from typing_extensions import TypedDict, TypeIs
+
+from dataeval_flow.config.schemas import ResultMetadata
+from dataeval_flow.workflow.base import Reportable, WorkflowOutputsBase, WorkflowReportBase
+
+if TYPE_CHECKING:
+    from dataeval_flow.workflow import WorkflowResult
+
+__all__ = [
+    "ChunkResultDict",
+    "ClasswiseDriftDict",
+    "ClasswiseDriftRowDict",
+    "DetectorResultDict",
+    "DriftMonitoringMetadata",
+    "DriftMonitoringOutputs",
+    "DriftMonitoringRawOutputs",
+    "DriftMonitoringReport",
+    "DriftMonitoringResult",
+    "is_drift_result",
+]
+
+
+# ---------------------------------------------------------------------------
+# TypedDicts for serialized detector outputs
+# ---------------------------------------------------------------------------
+
+
+class ChunkResultDict(TypedDict):
+    """Single chunk result from chunked drift analysis."""
+
+    key: str  # e.g. "[0:100]"
+    index: int
+    start_index: int
+    end_index: int
+    value: float
+    upper_threshold: float | None
+    lower_threshold: float | None
+    drifted: bool
+
+
+class _DetectorResultRequired(TypedDict):
+    """Required fields for a detector result."""
+
+    method: str
+    drifted: bool
+    distance: float
+    threshold: float
+    metric_name: str
+
+
+class DetectorResultDict(_DetectorResultRequired, total=False):
+    """Serialized result from a single drift detector.
+
+    ``details`` contains detector-specific statistics (p-values, per-feature
+    flags, fold AUROCs, etc.).  ``chunks`` is present only when chunked
+    analysis is enabled.
+    """
+
+    details: dict[str, Any]
+    chunks: list[ChunkResultDict]
+
+
+class ClasswiseDriftRowDict(TypedDict):
+    """Per-class drift result for a single detector."""
+
+    class_name: str
+    drifted: bool
+    distance: float
+    p_val: float | None
+
+
+class ClasswiseDriftDict(TypedDict):
+    """Classwise drift results for one detector."""
+
+    detector: str
+    rows: list[ClasswiseDriftRowDict]
+
+
+# ---------------------------------------------------------------------------
+# Pydantic output models
+# ---------------------------------------------------------------------------
+
+
+class DriftMonitoringRawOutputs(WorkflowOutputsBase):
+    """Machine-readable results from drift monitoring workflow."""
+
+    reference_size: int = Field(
+        default=0,
+        description="Number of items in the reference dataset.",
+    )
+    test_size: int = Field(
+        default=0,
+        description="Number of items in the test dataset(s).",
+    )
+    detectors: dict[str, DetectorResultDict] = Field(
+        default_factory=dict,
+        description="Per-detector results keyed by method name.",
+    )
+    classwise: list[ClasswiseDriftDict] | None = Field(
+        default=None,
+        description="Per-class drift results (one entry per detector). None if classwise disabled.",
+    )
+
+
+class DriftMonitoringReport(WorkflowReportBase):
+    """Human-readable report for drift monitoring workflow."""
+
+    findings: list[Reportable] = Field(default_factory=list)
+
+
+class DriftMonitoringOutputs(BaseModel):
+    """Complete drift monitoring workflow output."""
+
+    raw: DriftMonitoringRawOutputs
+    report: DriftMonitoringReport
+
+
+class DriftMonitoringMetadata(ResultMetadata):
+    """Metadata for the drift-monitoring workflow."""
+
+    mode: Literal["advisory", "preparatory"] = "advisory"
+    detectors_used: list[str] = Field(default_factory=list)
+    chunking_enabled: bool = False
+    classwise_enabled: bool = False
+
+
+# ---------------------------------------------------------------------------
+# Type alias and TypeIs guard for type narrowing
+# ---------------------------------------------------------------------------
+
+#: Fully typed result alias for the drift-monitoring workflow.
+DriftMonitoringResult: TypeAlias = "WorkflowResult[DriftMonitoringMetadata, DriftMonitoringOutputs]"
+
+
+def is_drift_result(
+    result: "WorkflowResult[Any, Any]",
+) -> TypeIs["WorkflowResult[DriftMonitoringMetadata, DriftMonitoringOutputs]"]:
+    """Narrow a generic ``WorkflowResult`` to a drift-monitoring result."""
+    return isinstance(result.metadata, DriftMonitoringMetadata)

dataeval_flow/workflows/drift/params.py

@@ -0,0 +1,332 @@
+"""Drift monitoring workflow parameters."""
+
+from collections.abc import Sequence
+from typing import Annotated, ClassVar, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from dataeval_flow.workflow.base import WorkflowParametersBase
+
+__all__ = [
+    "ChunkingConfig",
+    "DriftDetectorConfig",
+    "DriftDetectorDomainClassifier",
+    "DriftDetectorKNeighbors",
+    "DriftDetectorMMD",
+    "DriftDetectorUnivariate",
+    "DriftHealthThresholds",
+    "DriftMonitoringParameters",
+    "UpdateStrategyConfig",
+]
+
+
+# ---------------------------------------------------------------------------
+# Chunking configuration
+# ---------------------------------------------------------------------------
+
+
+class ChunkingConfig(BaseModel):
+    """Configuration for chunked (temporal) drift analysis.
+
+    When present, the test data is split into sequential chunks and each
+    chunk is tested for drift independently.  This answers "when did drift
+    start?" and "did it recover?".
+    """
+
+    chunk_size: int | None = Field(
+        default=None,
+        gt=0,
+        description="Fixed number of samples per chunk. Mutually exclusive with chunk_count.",
+    )
+    chunk_count: int | None = Field(
+        default=None,
+        gt=0,
+        description="Split test data into this many equal chunks. Mutually exclusive with chunk_size.",
+    )
+    incomplete: Literal["keep", "drop", "append"] = Field(
+        default="keep",
+        description=(
+            "How to handle a final chunk smaller than chunk_size. "
+            "'keep': retain as-is, 'drop': discard, 'append': merge into last full chunk."
+        ),
+    )
+    threshold_multiplier: float = Field(
+        default=3.0,
+        gt=0.0,
+        description=(
+            "Z-score multiplier for the chunk-level drift threshold. "
+            "Lower values are more sensitive (e.g. 2.0), higher values more conservative."
+        ),
+    )
+
+    @model_validator(mode="after")
+    def _validate_chunking(self) -> "ChunkingConfig":
+        if self.chunk_size is None and self.chunk_count is None:
+            raise ValueError("Either chunk_size or chunk_count must be set.")
+        if self.chunk_size is not None and self.chunk_count is not None:
+            raise ValueError("chunk_size and chunk_count are mutually exclusive.")
+        return self
+
+
+# ---------------------------------------------------------------------------
+# Drift detector configs — discriminated union on ``method``
+# ---------------------------------------------------------------------------
+
+
+class DriftDetectorUnivariate(BaseModel):
+    """Univariate statistical test per feature.
+
+    Applies a chosen statistical test independently to each feature dimension
+    and flags drift when enough features reject the null hypothesis (after
+    multiple-testing correction).
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    method: Literal["univariate"] = "univariate"
+    test: Literal["ks", "cvm", "mwu", "anderson", "bws"] = Field(
+        default="ks",
+        description=(
+            "Statistical test to apply per feature. "
+            "'ks' (Kolmogorov-Smirnov) is a good general-purpose default. "
+            "'cvm' (Cramér-von Mises) integrates squared CDF distance. "
+            "'mwu' (Mann-Whitney U) is rank-based and outlier-robust. "
+            "'anderson' (Anderson-Darling) is tail-sensitive. "
+            "'bws' (Baumgartner-Weiss-Schindler) has high power with tail sensitivity."
+        ),
+    )
+    p_val: float = Field(
+        default=0.05,
+        gt=0.0,
+        lt=1.0,
+        description="Significance level for the test.",
+    )
+    correction: Literal["bonferroni", "fdr"] = Field(
+        default="bonferroni",
+        description="Multiple-testing correction method across features.",
+    )
+    alternative: Literal["two-sided", "less", "greater"] = Field(
+        default="two-sided",
+        description="Alternative hypothesis direction.",
+    )
+    n_features: int | None = Field(
+        default=None,
+        gt=0,
+        description="Expected number of features. None = infer from reference data.",
+    )
+    classwise: bool = Field(
+        default=False,
+        description=(
+            "Run drift detection per class for this detector. Requires labelled datasets. Runs non-chunked only."
+        ),
+    )
+    chunking: ChunkingConfig | None = Field(
+        default=None,
+        description="Chunked (temporal) drift analysis for this detector. None disables chunking.",
+    )
+
+
+class DriftDetectorMMD(BaseModel):
+    """Maximum Mean Discrepancy with permutation test.
+
+    Computes the MMD² statistic between reference and test distributions
+    using an RBF kernel, then estimates a p-value via permutation testing.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    method: Literal["mmd"] = "mmd"
+    p_val: float = Field(
+        default=0.05,
+        gt=0.0,
+        lt=1.0,
+        description="Significance level for the permutation test.",
+    )
+    n_permutations: int = Field(
+        default=100,
+        gt=0,
+        description="Number of permutations for the test.",
+    )
+    device: str | None = Field(
+        default=None,
+        description="PyTorch device for kernel computation (e.g. 'cpu', 'cuda:0'). None = auto.",
+    )
+    classwise: bool = Field(
+        default=False,
+        description=(
+            "Run drift detection per class for this detector. Requires labelled datasets. Runs non-chunked only."
+        ),
+    )
+    chunking: ChunkingConfig | None = Field(
+        default=None,
+        description="Chunked (temporal) drift analysis for this detector. None disables chunking.",
+    )
+
+
+class DriftDetectorDomainClassifier(BaseModel):
+    """Binary domain classifier (LightGBM) approach.
+
+    Trains a binary classifier to distinguish reference from test data.
+    An AUROC significantly above 0.5 indicates the distributions are
+    distinguishable — i.e. drift has occurred.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    method: Literal["domain_classifier"] = "domain_classifier"
+    n_folds: int = Field(
+        default=5,
+        ge=2,
+        description="Number of cross-validation folds.",
+    )
+    threshold: float = Field(
+        default=0.55,
+        gt=0.5,
+        le=1.0,
+        description="AUROC threshold above which drift is declared.",
+    )
+    classwise: bool = Field(
+        default=False,
+        description=(
+            "Run drift detection per class for this detector. Requires labelled datasets. Runs non-chunked only."
+        ),
+    )
+    chunking: ChunkingConfig | None = Field(
+        default=None,
+        description="Chunked (temporal) drift analysis for this detector. None disables chunking.",
+    )
+
+
+class DriftDetectorKNeighbors(BaseModel):
+    """K-nearest neighbors distance-based drift detection.
+
+    Computes per-sample k-NN distances for test data against the reference
+    set. A Mann-Whitney U test determines whether test distances are
+    stochastically larger than reference self-distances.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    method: Literal["kneighbors"] = "kneighbors"
+    k: int = Field(
+        default=10,
+        gt=0,
+        description="Number of nearest neighbors.",
+    )
+    distance_metric: Literal["cosine", "euclidean"] = Field(
+        default="euclidean",
+        description="Distance metric for k-NN computation.",
+    )
+    p_val: float = Field(
+        default=0.05,
+        gt=0.0,
+        lt=1.0,
+        description="Significance level for the Mann-Whitney U test.",
+    )
+    classwise: bool = Field(
+        default=False,
+        description=(
+            "Run drift detection per class for this detector. Requires labelled datasets. Runs non-chunked only."
+        ),
+    )
+    chunking: ChunkingConfig | None = Field(
+        default=None,
+        description="Chunked (temporal) drift analysis for this detector. None disables chunking.",
+    )
+
+
+# Discriminated union — Pydantic selects the right model based on ``method``.
+DriftDetectorConfig = Annotated[
+    DriftDetectorUnivariate | DriftDetectorMMD | DriftDetectorDomainClassifier | DriftDetectorKNeighbors,
+    Field(discriminator="method"),
+]
+
+
+# ---------------------------------------------------------------------------
+# Update strategy (stubbed for future online detection)
+# ---------------------------------------------------------------------------
+
+
+class UpdateStrategyConfig(BaseModel):
+    """Reference-set update strategy — **stubbed for future use**.
+
+    When online drift detection is implemented, this will control how the
+    reference distribution evolves after each prediction step.  Currently
+    accepted in configuration but not applied at runtime.
+    """
+
+    type: Literal["last_seen", "reservoir_sampling"] = Field(
+        description="Update algorithm. 'last_seen': sliding window. 'reservoir_sampling': uniform random sample.",
+    )
+    n: int = Field(
+        gt=0,
+        description="Number of samples to retain in the updated reference set.",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Health thresholds
+# ---------------------------------------------------------------------------
+
+
+class DriftHealthThresholds(BaseModel):
+    """Configurable thresholds that control finding severity.
+
+    Findings that exceed a threshold are elevated to ``severity="warning"``;
+    otherwise they stay at ``severity="info"``.
+    """
+
+    any_drift_is_warning: bool = Field(
+        default=True,
+        description=(
+            "Non-chunked mode: if any detector flags drift, the finding is a warning. "
+            "Set to False to treat all non-chunked results as informational."
+        ),
+    )
+    chunk_drift_pct_warning: float = Field(
+        default=10.0,
+        ge=0.0,
+        le=100.0,
+        description="Chunked mode: percentage of chunks that must drift to trigger a warning.",
+    )
+    consecutive_chunks_warning: int = Field(
+        default=3,
+        ge=1,
+        description=(
+            "Chunked mode: number of consecutive drifted chunks that triggers a warning. "
+            "Sustained drift is typically more concerning than sporadic drift."
+        ),
+    )
+    classwise_any_drift_is_warning: bool = Field(
+        default=True,
+        description="Classwise mode: if any class drifts for any detector, the finding is a warning.",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Top-level parameters
+# ---------------------------------------------------------------------------
+
+
+class DriftMonitoringParameters(WorkflowParametersBase):
+    """Parameters for the drift-monitoring workflow.
+
+    At least one detector must be configured.  Chunking and classwise
+    analysis are optional extensions.
+    """
+
+    detectors: Sequence[DriftDetectorConfig] = Field(
+        min_length=1,
+        description="List of drift detectors to run. At least one required.",
+    )
+    update_strategy: UpdateStrategyConfig | None = Field(
+        default=None,
+        description=(
+            "Reference-set update strategy (stubbed — not yet applied at runtime). "
+            "Accepted in config to preserve forward compatibility."
+        ),
+    )
+    health_thresholds: DriftHealthThresholds = Field(
+        default_factory=DriftHealthThresholds,
+        description="Warning thresholds for drift severity classification.",
+    )

dataeval_flow/workflows/drift/report.py

@@ -0,0 +1,201 @@
+"""Findings builders for the drift monitoring workflow."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+import numpy as np
+
+from dataeval_flow.workflow.base import Reportable
+from dataeval_flow.workflows.drift.outputs import (
+    ChunkResultDict,
+    ClasswiseDriftRowDict,
+    DetectorResultDict,
+    DriftMonitoringRawOutputs,
+)
+from dataeval_flow.workflows.drift.params import (
+    DriftHealthThresholds,
+    DriftMonitoringParameters,
+)
+
+
+def _severity_for_detector(
+    drifted: bool,
+    thresholds: DriftHealthThresholds,
+) -> Literal["ok", "info", "warning"]:
+    """Determine severity for a non-chunked detector result."""
+    if drifted and thresholds.any_drift_is_warning:
+        return "warning"
+    return "info" if drifted else "ok"
+
+
+def _severity_for_chunks(
+    chunks: list[ChunkResultDict],
+    thresholds: DriftHealthThresholds,
+) -> Literal["ok", "info", "warning"]:
+    """Determine severity for chunked results."""
+    if not chunks:
+        return "ok"
+    n_drifted = sum(1 for c in chunks if c["drifted"])
+    pct = 100.0 * n_drifted / len(chunks) if chunks else 0.0
+
+    # Check consecutive drift window
+    max_consecutive = _max_consecutive_drifted(chunks)
+
+    if pct >= thresholds.chunk_drift_pct_warning:
+        return "warning"
+    if max_consecutive >= thresholds.consecutive_chunks_warning:
+        return "warning"
+    return "info" if n_drifted > 0 else "ok"
+
+
+def _max_consecutive_drifted(chunks: list[ChunkResultDict]) -> int:
+    """Count the longest run of consecutive drifted chunks."""
+    max_run = 0
+    current_run = 0
+    for c in chunks:
+        if c["drifted"]:
+            current_run += 1
+            max_run = max(max_run, current_run)
+        else:
+            current_run = 0
+    return max_run
+
+
+def _build_detector_finding(
+    name: str,
+    result: DetectorResultDict,
+    thresholds: DriftHealthThresholds,
+    classwise_rows: list[ClasswiseDriftRowDict] | None = None,
+) -> Reportable:
+    """Build a finding for a single detector (non-chunked)."""
+    drifted = result["drifted"]
+    severity = _severity_for_detector(drifted, thresholds)
+    data: dict[str, Any] = {
+        "distance": round(result["distance"], 4),
+        "threshold": round(result["threshold"], 4),
+        "metric": result["metric_name"],
+    }
+
+    # Add p_val from details if available
+    details = result.get("details", {})
+    if isinstance(details, dict) and "p_val" in details:
+        data["p_val"] = round(float(details["p_val"]), 6)
+
+    # Univariate: summarize feature drift
+    if isinstance(details, dict) and "feature_drift" in details:
+        fd = details["feature_drift"]
+        if isinstance(fd, list):
+            n_drifted = sum(fd)
+            n_total = len(fd)
+        else:
+            n_drifted = int(np.sum(fd))
+            n_total = len(fd)
+        data["features_drifted"] = f"{n_drifted} / {n_total}"
+
+    description = f"{name}: distance={data['distance']}, threshold={data['threshold']}"
+    if "p_val" in data:
+        description += f", p={data['p_val']}"
+
+    # Classwise breakdown → render as a classwise_table instead of key_value
+    if classwise_rows:
+        drifted_classes = [r["class_name"] for r in classwise_rows if r["drifted"]]
+        n_cls_drifted = len(drifted_classes)
+        n_total = len(classwise_rows)
+        description = f"Classes drifted: {', '.join(drifted_classes)}" if drifted_classes else "No classes drifted"
+        if n_cls_drifted > 0 and thresholds.classwise_any_drift_is_warning:
+            severity = "warning"
+
+        table_rows = [
+            {
+                "Class": r["class_name"],
+                "Distance": round(r["distance"], 4),
+                "PVal": round(r["p_val"], 6) if r["p_val"] is not None else None,
+                "Status": "DRIFT" if r["drifted"] else "ok",
+            }
+            for r in classwise_rows
+        ]
+
+        data["table_rows"] = table_rows
+        data["brief"] = f"{n_cls_drifted}/{n_total} classes drifted"
+
+        return Reportable(
+            report_type="classwise_table",
+            severity=severity,
+            title=name,
+            data=data,
+            description=description,
+        )
+
+    return Reportable(
+        report_type="key_value",
+        severity=severity,
+        title=name,
+        data=data,
+        description=description,
+    )
+
+
+def _build_chunked_finding(
+    name: str,
+    result: DetectorResultDict,
+    thresholds: DriftHealthThresholds,
+) -> Reportable:
+    """Build a table finding for chunked detector results."""
+    chunks = result.get("chunks", [])
+    if not chunks:
+        return _build_detector_finding(name, result, thresholds)
+
+    severity = _severity_for_chunks(chunks, thresholds)
+    n_drifted = sum(1 for c in chunks if c["drifted"])
+    pct = 100.0 * n_drifted / len(chunks) if chunks else 0.0
+    max_consec = _max_consecutive_drifted(chunks)
+
+    rows: list[dict[str, Any]] = [
+        {
+            "Chunk": c["key"],
+            "Distance": round(c["value"], 4),
+            "UpperThreshold": round(c["upper_threshold"], 4) if c["upper_threshold"] is not None else None,
+            "LowerThreshold": round(c["lower_threshold"], 4) if c["lower_threshold"] is not None else None,
+            "Status": "DRIFT" if c["drifted"] else "ok",
+        }
+        for c in chunks
+    ]
+
+    description = f"{n_drifted}/{len(chunks)} chunks drifted ({pct:.0f}%) | max consecutive: {max_consec}"
+
+    return Reportable(
+        report_type="chunk_table",
+        severity=severity,
+        title=name,
+        data={
+            "table_rows": rows,
+            "drift_flags": [c["drifted"] for c in chunks],
+        },
+        description=description,
+    )
+
+
+def build_findings(
+    raw: DriftMonitoringRawOutputs,
+    params: DriftMonitoringParameters,
+    detector_names: dict[str, str],
+) -> list[Reportable]:
+    """Build all report findings from raw results."""
+    findings: list[Reportable] = []
+
+    # Index classwise results by detector display name for per-detector lookup
+    classwise_by_detector: dict[str, list[ClasswiseDriftRowDict]] = {}
+    if raw.classwise:
+        for cw in raw.classwise:
+            classwise_by_detector[cw["detector"]] = cw["rows"]
+
+    for method_key, result in raw.detectors.items():
+        name = detector_names.get(method_key, method_key)
+        cw_rows = classwise_by_detector.get(name)
+        if result.get("chunks"):
+            findings.append(_build_chunked_finding(name, result, params.health_thresholds))
+        else:
+            findings.append(_build_detector_finding(name, result, params.health_thresholds, cw_rows))
+
+    return findings