dataeval-flow 0.1.0__py3-none-any.whl

dataeval_flow/config/schemas/_extractor.py

@@ -0,0 +1,119 @@
+"""Extractor configuration schemas — one class per model type."""
+
+from typing import ClassVar, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from dataeval_flow.config._paths import validate_config_path
+
+
+class _ExtractorConfigBase(BaseModel):
+    """Common fields shared by all extractor model types."""
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    name: str = Field(description="Identifier for the extractor")
+    preprocessor: str | None = Field(default=None, description="Reference to a preprocessor name (optional)")
+    batch_size: int | None = Field(default=None, description="Batch size for embedding extraction")
+
+
+class OnnxExtractorConfig(_ExtractorConfigBase):
+    """Extractor config for ONNX models.
+
+    YAML example::
+
+        extractors:
+          - name: resnet_extractor
+            model: onnx
+            model_path: "./resnet50.onnx"
+            output_name: "flatten0"
+            preprocessor: resnet_preprocess
+            batch_size: 64
+    """
+
+    model: Literal["onnx"] = "onnx"
+    model_path: str = Field(description="Path to ONNX model file (relative to data root).")
+    output_name: str | None = Field(default=None, description="Output layer name.")
+    flatten: bool = Field(default=True, description="Flatten output to (N, D) shape.")
+
+    @field_validator("model_path")
+    @classmethod
+    def _model_path_must_be_relative(cls, v: str) -> str:
+        return validate_config_path(v)
+
+
+class BoVWExtractorConfig(_ExtractorConfigBase):
+    """Extractor config for Bag-of-Visual-Words.
+
+    YAML example::
+
+        extractors:
+          - name: bovw_extractor
+            model: bovw
+            vocab_size: 1024
+            batch_size: 32
+    """
+
+    model: Literal["bovw"] = "bovw"
+    vocab_size: int = Field(default=2048, ge=256, le=4096, description="Visual word count.")
+
+
+class FlattenExtractorConfig(_ExtractorConfigBase):
+    """Extractor config for simple flattening (no model).
+
+    YAML example::
+
+        extractors:
+          - name: flat_extractor
+            model: flatten
+    """
+
+    model: Literal["flatten"] = "flatten"
+
+
+class TorchExtractorConfig(_ExtractorConfigBase):
+    """Extractor config for PyTorch models.
+
+    YAML example::
+
+        extractors:
+          - name: torch_extractor
+            model: torch
+            model_path: "./resnet.pt"
+            layer_name: layer4
+            device: cpu
+    """
+
+    model: Literal["torch"] = "torch"
+    model_path: str = Field(description="Path to PyTorch model file (relative to data root).")
+    layer_name: str | None = Field(default=None, description="Layer for forward hook extraction.")
+    use_output: bool = Field(default=True, description="Capture layer output (True) or input (False).")
+    device: str | None = Field(default=None, description="Device (e.g., 'cpu', 'cuda:0').")
+
+    @field_validator("model_path")
+    @classmethod
+    def _model_path_must_be_relative(cls, v: str) -> str:
+        return validate_config_path(v)
+
+
+class UncertaintyExtractorConfig(_ExtractorConfigBase):
+    """Extractor config for uncertainty estimation models.
+
+    YAML example::
+
+        extractors:
+          - name: unc_extractor
+            model: uncertainty
+            model_path: "./classifier.pt"
+            preds_type: logits
+    """
+
+    model: Literal["uncertainty"] = "uncertainty"
+    model_path: str = Field(description="Path to model file (relative to data root).")
+    preds_type: Literal["probs", "logits"] | None = Field(default=None, description="Model output format.")
+    device: str | None = Field(default=None, description="Device (e.g., 'cpu', 'cuda:0').")
+
+    @field_validator("model_path")
+    @classmethod
+    def _model_path_must_be_relative(cls, v: str) -> str:
+        return validate_config_path(v)

dataeval_flow/config/schemas/_metadata.py

@@ -0,0 +1,28 @@
+"""Result metadata schema for JATIC compliance [IR-3-H-12]."""
+
+from collections.abc import Sequence
+from datetime import datetime, timezone
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ResultMetadata(BaseModel):
+    """Base metadata envelope for workflow results.
+
+    Contains JATIC-required fields (version, timestamp, tool info,
+    dataset identifiers).
+    """
+
+    version: str = "1.0"
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    dataset_id: str | Sequence[str] = ""
+    label_source: str | None = None
+    model_id: str | None = None
+    preprocessor_id: str | None = None
+    selection_id: str | None = None
+    source_descriptions: Sequence[str] = ()
+    resolved_config: dict[str, Any] = Field(default_factory=dict)
+    tool: str = "dataeval-flow"
+    tool_version: str = ""
+    execution_time_s: float | None = None

dataeval_flow/config/schemas/_preprocessor.py

@@ -0,0 +1,18 @@
+"""Preprocessor configuration schema."""
+
+from collections.abc import Sequence
+
+from pydantic import BaseModel
+
+from dataeval_flow.preprocessing import PreprocessingStep
+
+
+class PreprocessorConfig(BaseModel):
+    """Named preprocessor pipeline configuration.
+
+    Steps can be torchvision transforms (serializable, YAML-configurable)
+    or arbitrary callables (programmatic only).
+    """
+
+    name: str
+    steps: Sequence[PreprocessingStep]

dataeval_flow/config/schemas/_selection.py

@@ -0,0 +1,100 @@
+"""Selection configuration schema."""
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from pydantic import BaseModel, Field, model_validator
+
+_MAX_INDICES_RANGE = 1_000_000
+
+
+class SelectionStep(BaseModel):
+    """Single selection step - pass-through to dataeval.selection.
+
+    See: https://dataeval.readthedocs.io/en/latest/reference/autoapi/dataeval/selection/index.html
+
+    The ``indices`` param supports a range shorthand so that contiguous
+    index spans do not need to be enumerated in config files::
+
+        # Expanded form (still supported)
+        params:
+          indices: [500, 501, 502, ..., 549]
+
+        # Range shorthand
+        params:
+          indices: {start: 500, stop: 550}
+
+        # Range with step
+        params:
+          indices: {start: 0, stop: 100, step: 2}
+    """
+
+    type: str = Field(description="Selection class from dataeval.selection")
+    params: Mapping[str, Any] = Field(
+        default_factory=dict,
+        json_schema_extra={
+            "properties": {
+                "indices": {
+                    "anyOf": [
+                        {"type": "array", "items": {"type": "integer"}},
+                        {
+                            "type": "object",
+                            "properties": {
+                                "start": {"type": "integer"},
+                                "stop": {"type": "integer"},
+                                "step": {"type": "integer"},
+                            },
+                            "required": ["start", "stop"],
+                            "additionalProperties": False,
+                        },
+                    ],
+                    "description": "Indices as a list or {start, stop[, step]} range shorthand.",
+                },
+            },
+        },
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def _expand_range_params(cls, data: Any) -> Any:
+        """Expand ``indices: {start, stop[, step]}`` into a list of ints."""
+        if not isinstance(data, dict):  # pragma: no cover — Pydantic v2 rejects non-dict before reaching here
+            return data
+        params = data.get("params")
+        if not isinstance(params, dict):
+            return data
+        indices = params.get("indices")
+        if isinstance(indices, dict):
+            allowed = {"start", "stop", "step"}
+            extra = set(indices) - allowed
+            if extra:
+                raise ValueError(
+                    f"Invalid keys in indices range shorthand: {extra}. "
+                    f"Allowed keys are {allowed} (matching Python's range())."
+                )
+            if "start" not in indices or "stop" not in indices:
+                raise ValueError("indices range shorthand requires both 'start' and 'stop' keys.")
+            r = range(
+                indices["start"],
+                indices["stop"],
+                indices.get("step", 1),
+            )
+            if len(r) > _MAX_INDICES_RANGE:
+                raise ValueError(
+                    f"indices range expands to {len(r):,} elements "
+                    f"(max {_MAX_INDICES_RANGE:,}). Use a smaller range or load indices from a file."
+                )
+            params = dict(params)
+            params["indices"] = list(r)
+            data = {**data, "params": params}
+        return data
+
+
+class SelectionConfig(BaseModel):
+    """Named selection pipeline configuration.
+
+    Similar to PreprocessorConfig - defines reusable selection pipelines.
+    """
+
+    name: str
+    steps: Sequence[SelectionStep]

dataeval_flow/config/schemas/_task.py

@@ -0,0 +1,89 @@
+"""Task configuration schema."""
+
+from collections.abc import Sequence
+from typing import Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+AutoBinMethod = Literal["uniform_width", "uniform_count", "clusters"]
+
+
+class TaskConfig(BaseModel):
+    """Task/workflow configuration schema.
+
+    Tasks reference sources (dataset+selection bundles) and an optional
+    extractor (model+preprocessor+batch_size bundle) by name.
+    """
+
+    name: str
+    workflow: str  # reference to WorkflowConfig.name (e.g. "clean_zscore_stats")
+    enabled: bool = Field(default=True, description="Whether this task is included when running the pipeline.")
+    sources: str | Sequence[str]  # reference to SourceConfig.name
+    extractor: str | None = None  # reference to ExtractorConfig.name
+
+
+class MultiSourceTaskConfig(TaskConfig):
+    """TaskConfig subclass that validates multiple sources/datasets for drift/ood tasks.
+
+    Validates that at least two sources are specified (reference + test).
+    The ``workflow`` field references a workflow instance whose type must
+    be either ``drift-monitoring`` or ``ood-detection`` — enforced at runtime
+    by the orchestrator.
+    """
+
+    @model_validator(mode="after")
+    def _require_multiple_sources(self) -> "MultiSourceTaskConfig":
+        srcs = self.sources if isinstance(self.sources, list) else [self.sources]
+        if len(srcs) < 2:
+            raise ValueError(f"{self.workflow} requires at least 2 sources (reference + test), got {len(srcs)}: {srcs}")
+        return self
+
+
+class DataAnalysisTaskConfig(TaskConfig):
+    """Task config for ``data-analysis`` workflows.
+
+    A typed subclass of :class:`TaskConfig` that enables typed overloads
+    on :func:`~dataeval_flow.workflow.orchestrator.run_tasks`, returning a
+    result with full access to analysis-specific metadata.
+    """
+
+
+class DataCleaningTaskConfig(TaskConfig):
+    """Task config for ``data-cleaning`` workflows.
+
+    A typed subclass of :class:`TaskConfig` that enables typed overloads
+    on :func:`~dataeval_flow.workflow.orchestrator.run_tasks`, returning a
+    :class:`~dataeval_flow.workflows.cleaning.outputs.DataCleaningResult`
+    with full access to cleaning-specific metadata (``mode``,
+    ``clean_indices``, ``flagged_indices``, ``removed_count``).
+    """
+
+
+class DataSplittingTaskConfig(TaskConfig):
+    """Task config for ``data-splitting`` workflows."""
+
+
+class DriftMonitoringTaskConfig(MultiSourceTaskConfig):
+    """Task config that validates drift-monitoring constraints.
+
+    Validates that at least two sources are specified (reference + test).
+    The ``workflow`` field references a workflow instance whose type must
+    be ``drift-monitoring`` — enforced at runtime by the orchestrator.
+    """
+
+
+class OODDetectionTaskConfig(MultiSourceTaskConfig):
+    """Task config that validates OOD detection constraints.
+
+    Validates that at least two datasets are specified (reference + test).
+    The ``workflow`` field references a workflow instance whose type must
+    be ``ood-detection`` — enforced at runtime by the orchestrator.
+    """
+
+
+class DataPrioritizationTaskConfig(MultiSourceTaskConfig):
+    """Task config for ``data-prioritization`` workflows.
+
+    Requires at least two sources: a reference (labeled) dataset and one
+    or more additional datasets to prioritize for labeling.
+    """

dataeval_flow/config/schemas/_workflow.py

@@ -0,0 +1,135 @@
+"""Workflow configuration schemas — one class per workflow type."""
+
+from typing import Literal
+
+from pydantic import Field
+
+from dataeval_flow.workflows.analysis.params import DataAnalysisParameters
+from dataeval_flow.workflows.cleaning.params import DataCleaningParameters
+from dataeval_flow.workflows.drift.params import DriftMonitoringParameters
+from dataeval_flow.workflows.ood.params import OODDetectionParameters
+from dataeval_flow.workflows.prioritization.params import DataPrioritizationParameters
+from dataeval_flow.workflows.splitting.params import DataSplittingParameters
+
+
+class DataAnalysisWorkflowConfig(DataAnalysisParameters):
+    """Typed workflow configuration for ``data-analysis``.
+
+    Inherits all fields from :class:`DataAnalysisParameters` — no ``params``
+    nesting required.
+
+    Example YAML::
+
+        workflows:
+          - name: cppe5_analysis
+            type: data-analysis
+            outlier_method: zscore
+            outlier_flags: [dimension, pixel, visual]
+            balance: true
+            diversity_method: simpson
+    """
+
+    name: str = Field(description="Identifier for this workflow")
+    type: Literal["data-analysis"] = "data-analysis"
+
+
+class DataCleaningWorkflowConfig(DataCleaningParameters):
+    """Typed workflow configuration for ``data-cleaning``.
+
+    Inherits all fields from :class:`DataCleaningParameters` — no ``params``
+    nesting required.
+
+    Example YAML::
+
+        workflows:
+          - name: clean_zscore_stats
+            type: data-cleaning
+            outlier_method: zscore
+            outlier_flags: [pixel, visual]
+    """
+
+    name: str = Field(description="Identifier for this workflow")
+    type: Literal["data-cleaning"] = "data-cleaning"
+
+
+class DataSplittingWorkflowConfig(DataSplittingParameters):
+    """Typed workflow configuration for ``data-splitting``.
+
+    Inherits all fields from :class:`DataSplittingParameters` — no ``params``
+    nesting required.
+
+    Example YAML::
+
+        workflows:
+          - name: split_stratified
+            type: data-splitting
+            test_frac: 0.2
+            val_frac: 0.1
+            stratify: true
+    """
+
+    name: str = Field(description="Identifier for this workflow")
+    type: Literal["data-splitting"] = "data-splitting"
+
+
+class DriftMonitoringWorkflowConfig(DriftMonitoringParameters):
+    """Typed workflow configuration for ``drift-monitoring``.
+
+    Inherits all fields from :class:`DriftMonitoringParameters` — no ``params``
+    nesting required.
+
+    Example YAML::
+
+        workflows:
+          - name: drift_knn
+            type: drift-monitoring
+            detectors:
+              - method: kneighbors
+                k: 10
+    """
+
+    name: str = Field(description="Identifier for this workflow")
+    type: Literal["drift-monitoring"] = "drift-monitoring"
+
+
+class OODDetectionWorkflowConfig(OODDetectionParameters):
+    """Typed workflow configuration for ``ood-detection``.
+
+    Inherits all fields from :class:`OODDetectionParameters` — no ``params``
+    nesting required.
+
+    Example YAML::
+
+        workflows:
+          - name: ood_knn
+            type: ood-detection
+            detectors:
+              - method: kneighbors
+                k: 10
+    """
+
+    name: str = Field(description="Identifier for this workflow")
+    type: Literal["ood-detection"] = "ood-detection"
+
+
+class DataPrioritizationWorkflowConfig(DataPrioritizationParameters):
+    """Typed workflow configuration for ``data-prioritization``.
+
+    Inherits all fields from :class:`DataPrioritizationParameters` — no ``params``
+    nesting required.
+
+    Example YAML::
+
+        workflows:
+          - name: prioritize_knn
+            type: data-prioritization
+            method: knn
+            k: 10
+            order: hard_first
+            cleaning:
+              outlier_method: adaptive
+              outlier_flags: [dimension, pixel]
+    """
+
+    name: str = Field(description="Identifier for this workflow")
+    type: Literal["data-prioritization"] = "data-prioritization"