dataeval-flow 0.1.0__py3-none-any.whl

dataeval_flow/config/__init__.py

@@ -0,0 +1,80 @@
+"""Config layer — single public API for all configuration types.
+
+Import everything from here::
+
+    from dataeval_flow.config import PipelineConfig, OnnxExtractorConfig, ...
+"""
+
+__all__ = [
+    # Dataset configs
+    "CocoDatasetConfig",
+    "DatasetProtocolConfig",
+    "HuggingFaceDatasetConfig",
+    "ImageFolderDatasetConfig",
+    "YoloDatasetConfig",
+    # Extractor configs
+    "BoVWExtractorConfig",
+    "FlattenExtractorConfig",
+    "OnnxExtractorConfig",
+    "TorchExtractorConfig",
+    "UncertaintyExtractorConfig",
+    # Workflow configs
+    "DataAnalysisWorkflowConfig",
+    "DataCleaningWorkflowConfig",
+    "DataPrioritizationWorkflowConfig",
+    "DriftMonitoringWorkflowConfig",
+    "OODDetectionWorkflowConfig",
+    # Task configs
+    "DataAnalysisTaskConfig",
+    "DataCleaningTaskConfig",
+    "DataPrioritizationTaskConfig",
+    "DriftMonitoringTaskConfig",
+    "OODDetectionTaskConfig",
+    "TaskConfig",
+    # Composition / pipeline
+    "PipelineConfig",
+    "SourceConfig",
+    # Other schemas
+    "PreprocessorConfig",
+    "ResultMetadata",
+    "SelectionConfig",
+    "SelectionStep",
+    # Loader functions
+    "export_params_schema",
+    "load_config",
+    "load_config_folder",
+]
+
+from dataeval_flow.config._loader import (
+    export_params_schema,
+    load_config,
+    load_config_folder,
+)
+from dataeval_flow.config._models import PipelineConfig, SourceConfig
+from dataeval_flow.config.schemas import (
+    BoVWExtractorConfig,
+    CocoDatasetConfig,
+    DataAnalysisTaskConfig,
+    DataAnalysisWorkflowConfig,
+    DataCleaningTaskConfig,
+    DataCleaningWorkflowConfig,
+    DataPrioritizationTaskConfig,
+    DataPrioritizationWorkflowConfig,
+    DatasetProtocolConfig,
+    DriftMonitoringTaskConfig,
+    DriftMonitoringWorkflowConfig,
+    FlattenExtractorConfig,
+    HuggingFaceDatasetConfig,
+    ImageFolderDatasetConfig,
+    OnnxExtractorConfig,
+    OODDetectionTaskConfig,
+    OODDetectionWorkflowConfig,
+    PreprocessorConfig,
+    ResultMetadata,
+    SelectionConfig,
+    SelectionStep,
+    TaskConfig,
+    TorchExtractorConfig,
+    UncertaintyExtractorConfig,
+    YoloDatasetConfig,
+)

dataeval_flow/config/_loader.py

@@ -0,0 +1,79 @@
+"""Config loading - YAML/JSON single-file, multi-file merge, schema export."""
+
+import json
+import logging
+import os
+from pathlib import Path
+
+import yaml
+
+from dataeval_flow.config._models import PipelineConfig
+from dataeval_flow.config._paths import relativize_to_data_dir, validate_config_path
+
+__all__ = [
+    "export_params_schema",
+    "get_data_dir",
+    "load_config",
+    "load_config_folder",
+    "relativize_to_data_dir",
+    "resolve_path",
+    "validate_config_path",
+]
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+DEFAULT_DATA_DIR = Path(".")
+_DATAEVAL_DATA_ENV = "DATAEVAL_DATA"
+
+
+def get_data_dir(data_dir: Path | None = None) -> Path:
+    """Resolve the data root directory.
+
+    Priority: explicit argument > ``DATAEVAL_DATA`` env var > current directory.
+    """
+    if data_dir is not None:
+        return data_dir
+    return Path(os.environ.get(_DATAEVAL_DATA_ENV, str(DEFAULT_DATA_DIR)))
+
+
+def resolve_path(relative: str | Path, data_dir: Path | None = None) -> Path:
+    """Resolve a user-provided path against *data_dir*.
+
+    Absolute paths are returned as-is.  Relative paths are joined to
+    *data_dir* (which itself defaults via :func:`get_data_dir`).
+    """
+    p = Path(relative)
+    if p.is_absolute():
+        return p
+    root = data_dir if data_dir is not None else get_data_dir()
+    return root / p
+
+
+def load_config(config_path: Path) -> PipelineConfig:
+    """Load pipeline configuration from a single YAML or JSON file."""
+    logger.debug("Loading config from %s", config_path)
+
+    if not config_path.exists():
+        msg = f"Config file not found: {config_path}"
+        raise FileNotFoundError(msg)
+
+    with open(config_path, encoding="utf-8") as f:
+        data = json.load(f) if config_path.suffix.lower() == ".json" else yaml.safe_load(f) or {}
+
+    return PipelineConfig.model_validate(data)
+
+
+def load_config_folder(config_path: Path) -> PipelineConfig:
+    """Load and merge all YAML/JSON files from config folder."""
+    from dataeval_flow.config._merge import merge_config_folder
+
+    logger.debug("Loading config folder %s", config_path)
+    merged = merge_config_folder(config_path)
+    return PipelineConfig.model_validate(merged)
+
+
+def export_params_schema(output_path: Path) -> None:
+    """Export JSON Schema for params.yaml (PipelineConfig) for IDE validation."""
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    schema = PipelineConfig.model_json_schema()
+    output_path.write_text(json.dumps(schema, indent=2), encoding="utf-8")

dataeval_flow/config/_merge.py

@@ -0,0 +1,92 @@
+"""Multi-file YAML/JSON configuration loader with schema validation."""
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+_YAML_EXTS = frozenset({".yaml", ".yml"})
+_JSON_EXTS = frozenset({".json"})
+_CONFIG_EXTS = _YAML_EXTS | _JSON_EXTS
+
+
+def _load_file(path: Path) -> dict[str, Any] | list[str]:
+    """Load a single YAML or JSON file and return its contents as a dict."""
+    with open(path, encoding="utf-8") as f:
+        return json.load(f) if path.suffix.lower() in _JSON_EXTS else yaml.safe_load(f) or []
+
+
+def _is_valid_config(data: dict[str, Any]) -> bool:
+    """Return True if *data* looks like a pipeline config fragment.
+
+    A file is accepted when it contains at least one key that is a known
+    ``PipelineConfig`` field **and** no keys that are completely unknown.
+    This allows partial configs (e.g. only ``datasets:``) while rejecting
+    unrelated files like JSON schemas.
+    """
+    from dataeval_flow.config._models import PipelineConfig
+
+    known_keys = set(PipelineConfig.model_fields)
+    file_keys = set(data.keys())
+    return bool(file_keys) and file_keys <= known_keys
+
+
+def merge_config_folder(config_path: Path) -> dict[str, Any]:
+    """Scan folder, merge all valid config files alphabetically.
+
+    Each candidate file is validated against the ``PipelineConfig`` schema
+    individually.  Files that fail validation (e.g. JSON schemas, unrelated
+    YAML) are silently skipped.  If no valid files are found, a
+    ``FileNotFoundError`` is raised.
+
+    Files are loaded in sorted order (00-base.yaml before 01-datasets.yaml).
+    Later files override earlier ones for duplicate keys.
+
+    Returns raw dict - use load_config_folder() for validated PipelineConfig.
+    """
+    config: dict[str, Any] = {}
+
+    if not config_path.is_dir():
+        raise ValueError(f"Config path is not a directory: {config_path}")
+
+    candidates = sorted(f for f in config_path.iterdir() if f.is_file() and f.suffix.lower() in _CONFIG_EXTS)
+    logger.debug("Found %d candidate file(s): %s", len(candidates), [f.name for f in candidates])
+
+    accepted: list[Path] = []
+    for config_file in candidates:
+        try:
+            file_config = _load_file(config_file)
+        except (json.JSONDecodeError, yaml.YAMLError) as exc:
+            logger.debug("Skipping %s (parse error: %s)", config_file.name, exc)
+            continue
+
+        if not isinstance(file_config, dict) or not _is_valid_config(file_config):
+            logger.debug("Skipping %s (not a valid pipeline config)", config_file.name)
+            continue
+
+        _deep_merge(config, file_config)
+        accepted.append(config_file)
+
+    if not accepted:
+        raise FileNotFoundError(f"No valid pipeline config files found in {config_path}")
+
+    logger.debug("Accepted %d config file(s): %s", len(accepted), [f.name for f in accepted])
+    return config
+
+
+def _deep_merge(base: dict, overlay: dict) -> None:
+    """Recursively merge overlay into base.
+
+    Rules: dicts merge recursively, lists extend, scalars replace.
+    """
+    for key, value in overlay.items():
+        if key in base and isinstance(base[key], dict) and isinstance(value, dict):
+            _deep_merge(base[key], value)
+        elif key in base and isinstance(base[key], list) and isinstance(value, list):
+            base[key].extend(value)
+        else:
+            base[key] = value

dataeval_flow/config/_models.py

@@ -0,0 +1,115 @@
+"""Pipeline and workflow composition models — SourceConfig, PipelineConfig."""
+
+__all__ = [
+    "PipelineConfig",
+    "SourceConfig",
+]
+
+from collections.abc import Sequence
+from typing import Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+from dataeval_flow.config.schemas import (
+    DatasetConfig,
+    DatasetProtocolConfig,
+    ExtractorConfig,
+    PreprocessorConfig,
+    SelectionConfig,
+    TaskConfig,
+    WorkflowConfig,
+)
+
+# ---------------------------------------------------------------------------
+# Source — dataset + optional selection
+# ---------------------------------------------------------------------------
+
+
+class SourceConfig(BaseModel):
+    """Named source definition — bundles a dataset with an optional selection.
+
+    YAML example::
+
+        sources:
+          - name: cifar_train_subset
+            dataset: cifar10_train
+            selection: first_5k
+    """
+
+    name: str = Field(description="Identifier for the source")
+    dataset: str = Field(description="Reference to a dataset name")
+    selection: str | None = Field(default=None, description="Reference to a selection name (optional)")
+
+
+# ---------------------------------------------------------------------------
+# Logging
+# ---------------------------------------------------------------------------
+
+
+class LoggingConfig(BaseModel):
+    """Logging level configuration."""
+
+    app_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "DEBUG"
+    lib_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "WARNING"
+
+
+# ---------------------------------------------------------------------------
+# Pipeline (top-level)
+# ---------------------------------------------------------------------------
+
+
+class PipelineConfig(BaseModel):
+    """Top-level pipeline configuration.
+
+    All sections use a define-once, reference-by-name pattern.
+    Sources compose datasets with optional selections; extractors
+    compose model type/params with optional preprocessors.
+    Tasks reference workflows, sources, and extractors by name.
+    """
+
+    # Logging
+    logging: LoggingConfig | None = None
+
+    # Named resource pools
+    datasets: Sequence[DatasetConfig | DatasetProtocolConfig] | None = None
+    preprocessors: Sequence[PreprocessorConfig] | None = None
+    selections: Sequence[SelectionConfig] | None = None
+
+    # Composition layers
+    sources: Sequence[SourceConfig] | None = Field(
+        default=None,
+        description="Named source definitions (dataset + optional selection)",
+    )
+    extractors: Sequence[ExtractorConfig] | None = Field(
+        default=None,
+        description="Named extractor definitions (model type + params + optional preprocessor + batch_size)",
+    )
+
+    # Execution
+    workflows: Sequence[WorkflowConfig] | None = Field(
+        default=None,
+        description="Named workflow configurations (type + params), referenced by tasks",
+    )
+    tasks: Sequence[TaskConfig] | None = None
+
+    @model_validator(mode="after")
+    def _check_unique_names(self) -> "PipelineConfig":
+        """Raise if any section contains duplicate names."""
+        sections: dict[str, Sequence | None] = {
+            "datasets": self.datasets,
+            "preprocessors": self.preprocessors,
+            "selections": self.selections,
+            "sources": self.sources,
+            "extractors": self.extractors,
+            "workflows": self.workflows,
+            "tasks": self.tasks,
+        }
+        for section_name, items in sections.items():
+            if items is None:
+                continue
+            seen: set[str] = set()
+            for item in items:
+                if item.name in seen:
+                    raise ValueError(f"Duplicate name '{item.name}' in {section_name}")
+                seen.add(item.name)
+        return self

dataeval_flow/config/_paths.py

@@ -0,0 +1,85 @@
+"""Config path validation and relativization utilities.
+
+These are separated from ``_loader`` to avoid circular imports — the
+schema modules (``_dataset``, ``_extractor``) need them at import time,
+while ``_loader`` depends on ``_models`` which depends on schemas.
+"""
+
+import os
+from pathlib import Path, PurePosixPath
+
+__all__ = [
+    "relativize_to_data_dir",
+    "validate_config_path",
+]
+
+
+def validate_config_path(value: str) -> str:
+    """Validate that a config path is relative and stays under the data root.
+
+    Config paths (dataset ``path``, extractor ``model_path``) must be
+    stored as relative paths so that configs remain portable across
+    machines and into containers where the data root is mounted at an
+    arbitrary location (e.g. ``/dataeval``).
+
+    Raises
+    ------
+    ValueError
+        If *value* is an absolute path or escapes upward via ``..``.
+    """
+    if not value:
+        raise ValueError("Path must not be empty")
+
+    p = PurePosixPath(value)
+    if p.is_absolute():
+        raise ValueError(
+            f"Config paths must be relative (got absolute path '{value}'). "
+            "Use a path relative to the data root instead."
+        )
+
+    # Normalize and check for upward escape via ".."
+    normalized = os.path.normpath(value)
+    if normalized.startswith(".."):
+        raise ValueError(
+            f"Config path '{value}' escapes the data root directory. "
+            "All paths must resolve to locations under the data root."
+        )
+
+    return value
+
+
+def relativize_to_data_dir(absolute_path: str | Path, data_dir: Path) -> str:
+    """Convert an absolute path to a relative path under *data_dir*.
+
+    Used by the TUI browse to convert OS-absolute paths into portable
+    relative paths suitable for config storage.
+
+    Parameters
+    ----------
+    absolute_path : str | Path
+        The absolute path to relativize.
+    data_dir : Path
+        The data root to relativize against.
+
+    Returns
+    -------
+    str
+        A relative path string under *data_dir*.
+
+    Raises
+    ------
+    ValueError
+        If *absolute_path* is not under *data_dir*.
+    """
+    p = Path(absolute_path).resolve()
+    root = data_dir.resolve()
+
+    try:
+        rel = p.relative_to(root)
+    except ValueError:
+        raise ValueError(
+            f"Path '{absolute_path}' is not under the data root '{root}'. "
+            "Config paths must point to locations inside the data directory."
+        ) from None
+
+    return str(rel)

dataeval_flow/config/schemas/__init__.py

@@ -0,0 +1,112 @@
+"""Schema catalog — concrete types and discriminated-union aliases.
+
+Private ``_*.py`` modules define the concrete schema classes.  This
+``__init__`` re-exports them and defines the discriminated-union type
+aliases (``DatasetConfig``, ``ExtractorConfig``, ``WorkflowConfig``)
+consumed by :class:`~dataeval_flow.config._models.PipelineConfig`.
+"""
+
+from typing import Annotated
+
+from pydantic import Field
+
+from dataeval_flow.config.schemas._dataset import (
+    CocoDatasetConfig,
+    DatasetProtocolConfig,
+    HuggingFaceDatasetConfig,
+    ImageFolderDatasetConfig,
+    YoloDatasetConfig,
+)
+from dataeval_flow.config.schemas._extractor import (
+    BoVWExtractorConfig,
+    FlattenExtractorConfig,
+    OnnxExtractorConfig,
+    TorchExtractorConfig,
+    UncertaintyExtractorConfig,
+)
+from dataeval_flow.config.schemas._metadata import ResultMetadata
+from dataeval_flow.config.schemas._preprocessor import PreprocessorConfig
+from dataeval_flow.config.schemas._selection import SelectionConfig, SelectionStep
+from dataeval_flow.config.schemas._task import (
+    AutoBinMethod,
+    DataAnalysisTaskConfig,
+    DataCleaningTaskConfig,
+    DataPrioritizationTaskConfig,
+    DataSplittingTaskConfig,
+    DriftMonitoringTaskConfig,
+    OODDetectionTaskConfig,
+    TaskConfig,
+)
+from dataeval_flow.config.schemas._workflow import (
+    DataAnalysisWorkflowConfig,
+    DataCleaningWorkflowConfig,
+    DataPrioritizationWorkflowConfig,
+    DataSplittingWorkflowConfig,
+    DriftMonitoringWorkflowConfig,
+    OODDetectionWorkflowConfig,
+)
+
+# -- discriminated-union aliases (internal) ---------------------------------
+
+DatasetConfig = Annotated[
+    HuggingFaceDatasetConfig | ImageFolderDatasetConfig | CocoDatasetConfig | YoloDatasetConfig,
+    Field(discriminator="format"),
+]
+
+ExtractorConfig = Annotated[
+    OnnxExtractorConfig
+    | BoVWExtractorConfig
+    | FlattenExtractorConfig
+    | TorchExtractorConfig
+    | UncertaintyExtractorConfig,
+    Field(discriminator="model"),
+]
+
+WorkflowConfig = Annotated[
+    DataAnalysisWorkflowConfig
+    | DataCleaningWorkflowConfig
+    | DataPrioritizationWorkflowConfig
+    | DataSplittingWorkflowConfig
+    | DriftMonitoringWorkflowConfig
+    | OODDetectionWorkflowConfig,
+    Field(discriminator="type"),
+]
+
+__all__ = [
+    # Dataset
+    "CocoDatasetConfig",
+    "DatasetConfig",
+    "DatasetProtocolConfig",
+    "HuggingFaceDatasetConfig",
+    "ImageFolderDatasetConfig",
+    "YoloDatasetConfig",
+    # Extractor
+    "BoVWExtractorConfig",
+    "ExtractorConfig",
+    "FlattenExtractorConfig",
+    "OnnxExtractorConfig",
+    "TorchExtractorConfig",
+    "UncertaintyExtractorConfig",
+    # Workflow
+    "DataAnalysisWorkflowConfig",
+    "DataCleaningWorkflowConfig",
+    "DataPrioritizationWorkflowConfig",
+    "DataSplittingWorkflowConfig",
+    "DriftMonitoringWorkflowConfig",
+    "OODDetectionWorkflowConfig",
+    "WorkflowConfig",
+    # Task
+    "AutoBinMethod",
+    "DataAnalysisTaskConfig",
+    "DataCleaningTaskConfig",
+    "DataPrioritizationTaskConfig",
+    "DataSplittingTaskConfig",
+    "DriftMonitoringTaskConfig",
+    "OODDetectionTaskConfig",
+    "TaskConfig",
+    # Other
+    "PreprocessorConfig",
+    "ResultMetadata",
+    "SelectionConfig",
+    "SelectionStep",
+]

dataeval_flow/config/schemas/_dataset.py

@@ -0,0 +1,111 @@
+"""Dataset configuration schemas — one class per format."""
+
+from typing import Any, ClassVar, Literal
+
+from pydantic import BaseModel, ConfigDict, field_validator
+
+from dataeval_flow.config._paths import validate_config_path
+
+
+class _DatasetConfigBase(BaseModel):
+    """Common fields shared by all dataset formats."""
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    format: Any
+    name: str
+    path: str
+
+    @field_validator("path")
+    @classmethod
+    def _path_must_be_relative(cls, v: str) -> str:
+        return validate_config_path(v)
+
+
+class HuggingFaceDatasetConfig(_DatasetConfigBase):
+    """Dataset config for HuggingFace format.
+
+    YAML example::
+
+        datasets:
+          - name: cifar10_train
+            format: huggingface
+            path: ./cifar10
+            split: train
+    """
+
+    format: Literal["huggingface"] = "huggingface"
+    split: str | None = None
+
+
+class ImageFolderDatasetConfig(_DatasetConfigBase):
+    """Dataset config for image_folder format.
+
+    YAML example::
+
+        datasets:
+          - name: photos
+            format: image_folder
+            path: photos
+            recursive: true
+            infer_labels: true
+    """
+
+    format: Literal["image_folder"] = "image_folder"
+    recursive: bool = False
+    infer_labels: bool = False
+
+
+class CocoDatasetConfig(_DatasetConfigBase):
+    """Dataset config for COCO format.
+
+    YAML example::
+
+        datasets:
+          - name: coco_train
+            format: coco
+            path: coco
+            annotations_file: instances_train.json
+            images_dir: train2017
+    """
+
+    format: Literal["coco"] = "coco"
+    annotations_file: str | None = None
+    images_dir: str | None = None
+    classes_file: str | None = None
+
+
+class YoloDatasetConfig(_DatasetConfigBase):
+    """Dataset config for YOLO format.
+
+    YAML example::
+
+        datasets:
+          - name: yolo_train
+            format: yolo
+            path: yolo
+            images_dir: images
+            labels_dir: labels
+            classes_file: classes.txt
+    """
+
+    format: Literal["yolo"] = "yolo"
+    images_dir: str | None = None
+    labels_dir: str | None = None
+    classes_file: str | None = None
+
+
+class DatasetProtocolConfig(BaseModel):
+    """Dataset Configuration schema for an in-memory dataset.
+
+    Not serializable — for programmatic use only. Cannot be loaded from
+    YAML/JSON config files or edited in the builder UI.
+    """
+
+    serializable: ClassVar[bool] = False
+    model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True)
+
+    name: str
+    format: Literal["maite", "torchvision"] = "maite"
+    dataset: Any
+    version: str = "1"