dataeval-flow 0.1.0__py3-none-any.whl

dataeval_flow/__init__.py

@@ -0,0 +1,93 @@
+"""DataEval Workflows - Data evaluation and monitoring pipelines.
+
+Quick start::
+
+    from pathlib import Path
+    from dataeval_flow import load_config, run_tasks
+
+    config = load_config(Path("/path/to/data/config.yaml"))
+    results = run_tasks(config, data_dir=Path("/path/to/data"))
+    print(results[0].report())  # text report
+    results[0].export("output/")  # write result JSON
+
+Or build a pipeline programmatically::
+
+    from dataeval_flow import (
+        PipelineConfig, HuggingFaceDatasetConfig, FlattenExtractorConfig,
+        SourceConfig, DataCleaningWorkflowConfig, TaskConfig, run_tasks,
+    )
+
+Discovery helpers::
+
+    >>> from dataeval_flow import list_workflows
+    >>> list_workflows()
+    [{'name': 'data-cleaning', ...}, {'name': 'drift-monitoring', ...}]
+"""
+
+from dataeval_flow.config import (
+    BoVWExtractorConfig,
+    CocoDatasetConfig,
+    DataCleaningTaskConfig,
+    DataCleaningWorkflowConfig,
+    DatasetProtocolConfig,
+    DriftMonitoringTaskConfig,
+    DriftMonitoringWorkflowConfig,
+    FlattenExtractorConfig,
+    HuggingFaceDatasetConfig,
+    ImageFolderDatasetConfig,
+    OnnxExtractorConfig,
+    PipelineConfig,
+    PreprocessorConfig,
+    SelectionConfig,
+    SelectionStep,
+    SourceConfig,
+    TaskConfig,
+    TorchExtractorConfig,
+    UncertaintyExtractorConfig,
+    YoloDatasetConfig,
+    export_params_schema,
+    load_config,
+    load_config_folder,
+)
+from dataeval_flow.dataset import load_dataset
+from dataeval_flow.workflow import WorkflowResult, get_workflow, list_workflows, run_tasks
+
+__all__ = [
+    # --- Core workflow ---
+    "load_config",
+    "load_config_folder",
+    "run_tasks",
+    "PipelineConfig",
+    "WorkflowResult",
+    # --- Discovery ---
+    "list_workflows",
+    "get_workflow",
+    # --- Dataset configs ---
+    "HuggingFaceDatasetConfig",
+    "ImageFolderDatasetConfig",
+    "CocoDatasetConfig",
+    "YoloDatasetConfig",
+    "DatasetProtocolConfig",
+    # --- Extractor configs ---
+    "OnnxExtractorConfig",
+    "BoVWExtractorConfig",
+    "FlattenExtractorConfig",
+    "TorchExtractorConfig",
+    "UncertaintyExtractorConfig",
+    # --- Workflow configs ---
+    "DataCleaningWorkflowConfig",
+    "DriftMonitoringWorkflowConfig",
+    # --- Task configs ---
+    "TaskConfig",
+    "DataCleaningTaskConfig",
+    "DriftMonitoringTaskConfig",
+    # --- Composition ---
+    "SourceConfig",
+    "PreprocessorConfig",
+    "SelectionConfig",
+    "SelectionStep",
+    # --- Utilities ---
+    "load_dataset",
+    "export_params_schema",
+]
+__version__ = "0.1.0"

dataeval_flow/__main__.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""CLI entry point for standalone usage: python -m dataeval_flow."""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+from typing import NoReturn
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    """Build the argument parser with subcommands."""
+    parser = argparse.ArgumentParser(
+        prog="dataeval_flow",
+        description="DataEval Flow - Data evaluation and monitoring pipelines",
+    )
+
+    # Headless execution flags (top-level, no subcommand needed)
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Increase verbosity: -v text report, -vv +INFO logs, -vvv +DEBUG logs.",
+    )
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=Path,
+        default=None,
+        help="Path to config file or folder. If omitted, auto-discovers YAML/JSON at the data root.",
+    )
+
+    _data_default = os.environ.get("DATAEVAL_DATA")
+    parser.add_argument(
+        "-d",
+        "--data",
+        type=Path,
+        default=Path(_data_default) if _data_default else None,
+        help="Root directory for data files (default: $DATAEVAL_DATA or current directory)",
+    )
+
+    _output_default = os.environ.get("DATAEVAL_OUTPUT")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        default=Path(_output_default) if _output_default else None,
+        help="Path to output directory for artifacts (default: $DATAEVAL_OUTPUT or None).",
+    )
+
+    _cache_default = os.environ.get("DATAEVAL_CACHE")
+    parser.add_argument(
+        "-k",
+        "--cache",
+        type=Path,
+        default=Path(_cache_default) if _cache_default else None,
+        help="Directory for disk-backed computation cache (default: $DATAEVAL_CACHE or None).",
+    )
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    # --- app (interactive TUI) ---
+    app_parser = subparsers.add_parser(
+        "app",
+        help="Launch interactive TUI dashboard",
+        description="Launch the interactive TUI dashboard. Requires: pip install dataeval-flow[app]",
+    )
+    app_parser.add_argument(
+        "--config",
+        type=Path,
+        default=None,
+        help="Path to an existing config file or folder to load on startup",
+    )
+    app_parser.add_argument(
+        "-d",
+        "--data",
+        type=Path,
+        default=None,
+        help="Root directory for data files (default: $DATAEVAL_DATA or current directory)",
+    )
+    app_parser.add_argument(
+        "-k",
+        "--cache",
+        type=Path,
+        default=None,
+        help="Directory for disk-backed computation cache (embeddings, metadata, stats).",
+    )
+
+    # --- config (simple CLI builder) ---
+    config_parser = subparsers.add_parser(
+        "config",
+        help="Create or edit config files (simple CLI)",
+        description="Interactive CLI config builder. Create and edit pipeline config files.",
+    )
+    config_parser.add_argument(
+        "--config",
+        type=Path,
+        default=None,
+        help="Path to an existing config file or folder to load on startup",
+    )
+
+    return parser
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse CLI arguments."""
+    parser = _build_parser()
+    return parser.parse_args()
+
+
+def main() -> NoReturn:
+    """CLI entry point."""
+    args = parse_args()
+
+    if args.command == "app":
+        try:
+            from dataeval_flow._app.app import run_builder
+        except ImportError:
+            print("ERROR: The interactive TUI requires the 'app' extra.")
+            print("")
+            print("Install with:")
+            print("  pip install dataeval-flow[app]")
+            print("")
+            print("For the simple CLI config editor, use:")
+            print("  dataeval-flow config")
+            sys.exit(1)
+
+        run_builder(config_path=args.config, data_dir=args.data, cache_dir=args.cache)
+        sys.exit(0)
+
+    if args.command == "config":
+        from dataeval_flow._app.cli import run_cli_builder
+
+        run_cli_builder(config_path=args.config)
+        sys.exit(0)
+
+    # Headless execution (no subcommand)
+    try:
+        from dataeval_flow.runner import run
+
+        sys.exit(run(args.config, args.output, data_dir=args.data, verbosity=args.verbose, cache_dir=args.cache))
+    except (FileNotFoundError, ValueError, ImportError) as e:
+        print(f"ERROR: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

dataeval_flow/_app/__init__.py

@@ -0,0 +1,5 @@
+"""Interactive configuration builder for DataEval Workflows.
+
+The TUI (Textual) app requires ``pip install dataeval-flow[app]``.
+The lightweight CLI builder uses ``click`` (included with ``datasets``).
+"""

dataeval_flow/_app/_model/__init__.py

@@ -0,0 +1,5 @@
+"""Model layer for the configuration builder.
+
+Pure data structures, business logic, and schema introspection.
+No UI framework dependencies.
+"""

dataeval_flow/_app/_model/_coerce.py

@@ -0,0 +1,126 @@
+"""Value coercion and validation for the configuration builder.
+
+Pure functions that convert string inputs to typed values based on
+type hints or field descriptors.  No UI or state dependencies.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from dataeval_flow._app._model._introspect import FieldDescriptor, FieldKind
+
+__all__ = [
+    "coerce_field_value",
+    "coerce_value",
+    "validate_value",
+]
+
+
+def _split_type_alternatives(type_hint: str) -> list[str]:
+    """Split a type hint like ``"int | str"`` into alternatives."""
+    parts: list[str] = []
+    depth = 0
+    current: list[str] = []
+    for ch in type_hint:
+        if ch in ("(", "["):
+            depth += 1
+            current.append(ch)
+        elif ch in (")", "]"):
+            depth -= 1
+            current.append(ch)
+        elif ch == "|" and depth == 0:
+            parts.append("".join(current).strip())
+            current = []
+        else:
+            current.append(ch)
+    parts.append("".join(current).strip())
+    return [p for p in parts if p]
+
+
+def validate_value(value: str, hint: str) -> bool:
+    """Check whether *value* is compatible with *hint*."""
+    h = hint.strip()
+    if h in ("any", "str", "string"):
+        return True
+    if h == "bool":
+        return value.lower() in ("true", "false", "1", "0", "yes", "no")
+    if h == "int":
+        try:
+            int(value)
+            return True
+        except ValueError:
+            return False
+    if h == "float":
+        try:
+            float(value)
+            return True
+        except ValueError:
+            return False
+    if h.startswith("list") or h.startswith("tuple"):
+        try:
+            parsed = json.loads(value)
+            return isinstance(parsed, (list, tuple))
+        except (ValueError, TypeError):
+            return False
+    return True
+
+
+_COERCE_SCALAR: dict[str, type] = {"int": int, "float": float}
+
+
+def _try_scalar_coercion(value: str, alternatives: list[str]) -> tuple[bool, Any]:
+    """Try int/float coercion. Returns ``(matched, result)``."""
+    for alt in alternatives:
+        converter = _COERCE_SCALAR.get(alt)
+        if converter is not None:
+            try:
+                return True, converter(value)
+            except (ValueError, TypeError):
+                continue
+    return False, value
+
+
+def coerce_value(value: str, type_hint: str) -> Any:
+    """Coerce a string *value* according to *type_hint*.
+
+    JSON parsing is only attempted for complex types (list, dict).
+    """
+    if not value:
+        return value
+    alternatives = [a.strip() for a in _split_type_alternatives(type_hint)]
+
+    if {"str", "string"} & set(alternatives):
+        matched, result = _try_scalar_coercion(value, alternatives)
+        return result if matched else value
+
+    matched, result = _try_scalar_coercion(value, alternatives)
+    if matched:
+        return result
+    if "bool" in alternatives:
+        return value.lower() in ("true", "1", "yes")
+
+    try:
+        return json.loads(value)
+    except (ValueError, TypeError):
+        pass
+    return value
+
+
+def coerce_field_value(value: str, desc: FieldDescriptor) -> Any:
+    """Coerce *value* using a FieldDescriptor's kind."""
+    if not value:
+        return value
+    if desc.kind == FieldKind.INT:
+        return int(value)
+    if desc.kind == FieldKind.FLOAT:
+        return float(value)
+    if desc.kind == FieldKind.BOOL:
+        return value.lower() in ("true", "1", "yes")
+    if desc.kind in (FieldKind.LIST, FieldKind.NESTED):
+        try:
+            return json.loads(value)
+        except (ValueError, TypeError):
+            return value
+    return value

dataeval_flow/_app/_model/_discover.py

@@ -0,0 +1,171 @@
+"""Runtime discovery of available transforms and selection classes.
+
+Introspects torchvision.transforms.v2 and dataeval.selection to provide
+dropdown options and parameter schemas for the builder TUI.
+"""
+
+from __future__ import annotations
+
+import collections.abc
+import inspect
+import types
+import typing
+from dataclasses import dataclass, field
+from functools import lru_cache
+from typing import Any, get_args, get_origin
+
+
+@dataclass
+class ParamInfo:
+    """Describes a single constructor parameter for a transform or selection class."""
+
+    name: str
+    type_hint: str  # human-readable type string
+    required: bool
+    default: Any = None
+    choices: list[str] = field(default_factory=list)  # for Literal types
+
+
+_PRIMITIVE_NAMES: dict[type, str] = {int: "int", float: "float", bool: "bool", str: "str"}
+
+
+def _simplify_union(annotation: Any) -> tuple[str, list[str]]:
+    """Simplify a Union or Optional type annotation."""
+    args = [a for a in get_args(annotation) if a is not type(None)]
+    if len(args) == 1:
+        return _simplify_type(args[0])
+    parts = []
+    for a in args:
+        t, _ = _simplify_type(a)
+        parts.append(t)
+    return " | ".join(dict.fromkeys(parts)), []
+
+
+def _simplify_type(annotation: Any) -> tuple[str, list[str]]:
+    """Convert a type annotation to a human-readable string + optional choices."""
+    if annotation is inspect.Parameter.empty or annotation is Any:
+        return "any", []
+
+    if get_origin(annotation) is typing.Literal:
+        return "select", [str(v) for v in get_args(annotation)]
+
+    origin = get_origin(annotation)
+    if origin is typing.Union or isinstance(annotation, types.UnionType):
+        return _simplify_union(annotation)
+
+    if annotation in _PRIMITIVE_NAMES:
+        return _PRIMITIVE_NAMES[annotation], []
+
+    if origin in (list, tuple, collections.abc.Sequence):
+        inner_args = get_args(annotation)
+        if inner_args:
+            inner_t, choices = _simplify_type(inner_args[0])
+            if choices:
+                return "select", choices
+            return f"list[{inner_t}]", []
+        return "list", []
+
+    return getattr(annotation, "__name__", str(annotation)), []
+
+
+def _introspect_params(cls: type) -> list[ParamInfo]:
+    """Extract constructor parameters from a class."""
+    try:
+        sig = inspect.signature(cls.__init__)
+    except (ValueError, TypeError):
+        return []
+
+    params: list[ParamInfo] = []
+    for pname, param in sig.parameters.items():
+        if pname == "self":
+            continue
+        # Skip *args/**kwargs
+        if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
+            continue
+
+        required = param.default is inspect.Parameter.empty
+        default = None if required else param.default
+        type_hint, choices = _simplify_type(param.annotation)
+
+        params.append(
+            ParamInfo(
+                name=pname,
+                type_hint=type_hint,
+                required=required,
+                default=default,
+                choices=choices,
+            )
+        )
+
+    return params
+
+
+@lru_cache(maxsize=1)
+def list_transforms() -> list[str]:
+    """Return sorted names of available torchvision.transforms.v2 classes."""
+    from torchvision.transforms import v2
+
+    skip = {
+        "Transform",
+        "Compose",
+        "Identity",
+        "RandomApply",
+        "RandomChoice",
+        "RandomOrder",
+        "AutoAugmentPolicy",
+        "InterpolationMode",
+        "Lambda",
+    }
+    names = []
+    for name, obj in inspect.getmembers(v2):
+        if (
+            inspect.isclass(obj)
+            and not name.startswith("_")
+            and name[0].isupper()
+            and name not in skip
+            and callable(obj)
+        ):
+            names.append(name)
+    return sorted(set(names))
+
+
+@lru_cache(maxsize=1)
+def list_selection_classes() -> list[str]:
+    """Return sorted names of available dataeval.selection classes."""
+    from dataeval import selection
+    from dataeval.selection._select import Selection
+
+    skip = {"Selection", "Subselection", "SelectionStage", "Select"}
+    names = []
+    for name, obj in inspect.getmembers(selection):
+        if (
+            inspect.isclass(obj)
+            and not name.startswith("_")
+            and name not in skip
+            and issubclass(obj, Selection)
+            and obj is not Selection
+        ):
+            names.append(name)
+    return sorted(names)
+
+
+@lru_cache(maxsize=64)
+def get_transform_params(name: str) -> list[ParamInfo]:
+    """Get parameter info for a torchvision.transforms.v2 class."""
+    from torchvision.transforms import v2
+
+    cls = getattr(v2, name, None)
+    if cls is None:
+        return []
+    return _introspect_params(cls)
+
+
+@lru_cache(maxsize=64)
+def get_selection_params(name: str) -> list[ParamInfo]:
+    """Get parameter info for a dataeval.selection class."""
+    from dataeval import selection
+
+    cls = getattr(selection, name, None)
+    if cls is None:
+        return []
+    return _introspect_params(cls)

dataeval_flow/_app/_model/_execution.py

@@ -0,0 +1,108 @@
+"""Per-task execution state tracking for the dashboard.
+
+Ephemeral — not saved to config files.  Thread-safe since Textual
+workers run in background threads.
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from dataeval_flow.workflow import WorkflowResult
+
+__all__ = ["ExecutionState", "TaskExecution"]
+
+
+@dataclass
+class TaskExecution:
+    """Runtime state for a single task execution."""
+
+    task_name: str
+    status: Literal["idle", "running", "completed", "failed"] = "idle"
+    result: WorkflowResult[Any, Any] | None = None
+    error: str | None = None
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+
+    @property
+    def elapsed_s(self) -> float | None:
+        """Return elapsed seconds, or ``None`` if not completed."""
+        if self.started_at and self.finished_at:
+            return (self.finished_at - self.started_at).total_seconds()
+        return None
+
+
+class ExecutionState:
+    """Tracks per-task execution status.  Thread-safe."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._tasks: dict[str, TaskExecution] = {}
+
+    def mark_running(self, name: str) -> TaskExecution:
+        """Set task to running.  Clears any previous result."""
+        with self._lock:
+            entry = TaskExecution(
+                task_name=name,
+                status="running",
+                started_at=datetime.now(timezone.utc),
+            )
+            self._tasks[name] = entry
+            return entry
+
+    def mark_completed(self, name: str, result: WorkflowResult[Any, Any]) -> TaskExecution:
+        """Set task to completed with its result."""
+        with self._lock:
+            entry = self._tasks.get(name)
+            if entry is None:
+                entry = TaskExecution(task_name=name)
+                self._tasks[name] = entry
+            entry.status = "completed"
+            entry.result = result
+            entry.error = None
+            entry.finished_at = datetime.now(timezone.utc)
+            return entry
+
+    def mark_failed(self, name: str, error: str) -> TaskExecution:
+        """Set task to failed with an error message."""
+        with self._lock:
+            entry = self._tasks.get(name)
+            if entry is None:
+                entry = TaskExecution(task_name=name)
+                self._tasks[name] = entry
+            entry.status = "failed"
+            entry.error = error
+            entry.result = None
+            entry.finished_at = datetime.now(timezone.utc)
+            return entry
+
+    def get(self, name: str) -> TaskExecution | None:
+        """Return execution entry for *name*, or ``None``."""
+        with self._lock:
+            return self._tasks.get(name)
+
+    def clear(self, name: str | None = None) -> None:
+        """Clear execution state.  If *name* is ``None``, clear all."""
+        with self._lock:
+            if name is None:
+                self._tasks.clear()
+            else:
+                self._tasks.pop(name, None)
+
+    def entries(self) -> list[TaskExecution]:
+        """Return a snapshot of all execution entries."""
+        with self._lock:
+            return list(self._tasks.values())
+
+    def completed_results(self) -> list[tuple[str, WorkflowResult[Any, Any]]]:
+        """Return ``(name, result)`` pairs for all completed tasks."""
+        with self._lock:
+            return [
+                (e.task_name, e.result)
+                for e in self._tasks.values()
+                if e.status == "completed" and e.result is not None
+            ]