dataeval-flow 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. dataeval_flow/__init__.py +93 -0
  2. dataeval_flow/__main__.py +149 -0
  3. dataeval_flow/_app/__init__.py +5 -0
  4. dataeval_flow/_app/_model/__init__.py +5 -0
  5. dataeval_flow/_app/_model/_coerce.py +126 -0
  6. dataeval_flow/_app/_model/_discover.py +171 -0
  7. dataeval_flow/_app/_model/_execution.py +108 -0
  8. dataeval_flow/_app/_model/_introspect.py +280 -0
  9. dataeval_flow/_app/_model/_item.py +213 -0
  10. dataeval_flow/_app/_model/_registry.py +255 -0
  11. dataeval_flow/_app/_model/_state.py +322 -0
  12. dataeval_flow/_app/_model/_undo.py +61 -0
  13. dataeval_flow/_app/_panes/__init__.py +35 -0
  14. dataeval_flow/_app/_panes/_config_pane.py +173 -0
  15. dataeval_flow/_app/_panes/_result_pane.py +125 -0
  16. dataeval_flow/_app/_panes/_task_pane.py +91 -0
  17. dataeval_flow/_app/_panes/_widgets.py +111 -0
  18. dataeval_flow/_app/_screens/__init__.py +25 -0
  19. dataeval_flow/_app/_screens/_base.py +242 -0
  20. dataeval_flow/_app/_screens/_detail.py +333 -0
  21. dataeval_flow/_app/_screens/_model.py +102 -0
  22. dataeval_flow/_app/_screens/_params.py +80 -0
  23. dataeval_flow/_app/_screens/_pathpicker.py +68 -0
  24. dataeval_flow/_app/_screens/_section.py +621 -0
  25. dataeval_flow/_app/_screens/_settings.py +183 -0
  26. dataeval_flow/_app/_viewmodel/__init__.py +15 -0
  27. dataeval_flow/_app/_viewmodel/_builder_vm.py +272 -0
  28. dataeval_flow/_app/_viewmodel/_model_vm.py +70 -0
  29. dataeval_flow/_app/_viewmodel/_rendering.py +189 -0
  30. dataeval_flow/_app/_viewmodel/_result_vm.py +210 -0
  31. dataeval_flow/_app/_viewmodel/_section_vm.py +224 -0
  32. dataeval_flow/_app/app.py +742 -0
  33. dataeval_flow/_app/cli.py +592 -0
  34. dataeval_flow/_logging.py +102 -0
  35. dataeval_flow/cache.py +1355 -0
  36. dataeval_flow/config/__init__.py +80 -0
  37. dataeval_flow/config/_loader.py +79 -0
  38. dataeval_flow/config/_merge.py +92 -0
  39. dataeval_flow/config/_models.py +115 -0
  40. dataeval_flow/config/_paths.py +85 -0
  41. dataeval_flow/config/schemas/__init__.py +112 -0
  42. dataeval_flow/config/schemas/_dataset.py +111 -0
  43. dataeval_flow/config/schemas/_extractor.py +119 -0
  44. dataeval_flow/config/schemas/_metadata.py +28 -0
  45. dataeval_flow/config/schemas/_preprocessor.py +18 -0
  46. dataeval_flow/config/schemas/_selection.py +100 -0
  47. dataeval_flow/config/schemas/_task.py +89 -0
  48. dataeval_flow/config/schemas/_workflow.py +135 -0
  49. dataeval_flow/dataset.py +635 -0
  50. dataeval_flow/embeddings.py +135 -0
  51. dataeval_flow/metadata.py +48 -0
  52. dataeval_flow/preprocessing.py +141 -0
  53. dataeval_flow/py.typed +0 -0
  54. dataeval_flow/runner.py +118 -0
  55. dataeval_flow/selection.py +50 -0
  56. dataeval_flow/workflow/__init__.py +328 -0
  57. dataeval_flow/workflow/_text_report.py +511 -0
  58. dataeval_flow/workflow/base.py +69 -0
  59. dataeval_flow/workflow/orchestrator.py +454 -0
  60. dataeval_flow/workflows/__init__.py +1 -0
  61. dataeval_flow/workflows/analysis/__init__.py +38 -0
  62. dataeval_flow/workflows/analysis/outputs.py +202 -0
  63. dataeval_flow/workflows/analysis/params.py +114 -0
  64. dataeval_flow/workflows/analysis/workflow.py +1313 -0
  65. dataeval_flow/workflows/cleaning/__init__.py +23 -0
  66. dataeval_flow/workflows/cleaning/outputs.py +200 -0
  67. dataeval_flow/workflows/cleaning/params.py +160 -0
  68. dataeval_flow/workflows/cleaning/report.py +304 -0
  69. dataeval_flow/workflows/cleaning/workflow.py +794 -0
  70. dataeval_flow/workflows/drift/__init__.py +1 -0
  71. dataeval_flow/workflows/drift/outputs.py +144 -0
  72. dataeval_flow/workflows/drift/params.py +332 -0
  73. dataeval_flow/workflows/drift/report.py +201 -0
  74. dataeval_flow/workflows/drift/workflow.py +647 -0
  75. dataeval_flow/workflows/ood/__init__.py +1 -0
  76. dataeval_flow/workflows/ood/outputs.py +134 -0
  77. dataeval_flow/workflows/ood/params.py +161 -0
  78. dataeval_flow/workflows/ood/report.py +311 -0
  79. dataeval_flow/workflows/ood/workflow.py +728 -0
  80. dataeval_flow/workflows/prioritization/__init__.py +1 -0
  81. dataeval_flow/workflows/prioritization/outputs.py +122 -0
  82. dataeval_flow/workflows/prioritization/params.py +124 -0
  83. dataeval_flow/workflows/prioritization/report.py +117 -0
  84. dataeval_flow/workflows/prioritization/workflow.py +587 -0
  85. dataeval_flow/workflows/splitting/__init__.py +25 -0
  86. dataeval_flow/workflows/splitting/outputs.py +101 -0
  87. dataeval_flow/workflows/splitting/params.py +61 -0
  88. dataeval_flow/workflows/splitting/report.py +485 -0
  89. dataeval_flow/workflows/splitting/workflow.py +371 -0
  90. dataeval_flow-0.1.0.dist-info/METADATA +305 -0
  91. dataeval_flow-0.1.0.dist-info/RECORD +94 -0
  92. dataeval_flow-0.1.0.dist-info/WHEEL +4 -0
  93. dataeval_flow-0.1.0.dist-info/entry_points.txt +2 -0
  94. dataeval_flow-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,454 @@
1
+ """Task orchestration — config → execution bridge."""
2
+
3
+ __all__ = ["run_task", "run_tasks"]
4
+
5
+ import logging
6
+ import time
7
+ from collections.abc import Mapping, Sequence
8
+ from pathlib import Path
9
+ from typing import TYPE_CHECKING, Any, Protocol, TypeVar, overload, runtime_checkable
10
+
11
+ from pydantic import BaseModel
12
+
13
+ logger: logging.Logger = logging.getLogger(__name__)
14
+
15
+ if TYPE_CHECKING:
16
+ from dataeval_flow.config import PipelineConfig, SourceConfig, TaskConfig
17
+ from dataeval_flow.config.schemas import WorkflowConfig
18
+ from dataeval_flow.config.schemas._task import (
19
+ DataAnalysisTaskConfig,
20
+ DataCleaningTaskConfig,
21
+ DataPrioritizationTaskConfig,
22
+ DriftMonitoringTaskConfig,
23
+ OODDetectionTaskConfig,
24
+ )
25
+ from dataeval_flow.workflow import DatasetContext, WorkflowResult
26
+ from dataeval_flow.workflows.analysis.outputs import DataAnalysisMetadata, DataAnalysisOutputs
27
+ from dataeval_flow.workflows.cleaning.outputs import DataCleaningMetadata, DataCleaningOutputs
28
+ from dataeval_flow.workflows.drift.outputs import DriftMonitoringMetadata, DriftMonitoringOutputs
29
+ from dataeval_flow.workflows.ood.outputs import OODDetectionMetadata, OODDetectionOutputs
30
+ from dataeval_flow.workflows.prioritization.outputs import DataPrioritizationMetadata, DataPrioritizationOutputs
31
+
32
+
33
+ @runtime_checkable
34
+ class _Named(Protocol):
35
+ """Protocol for config objects with a name attribute."""
36
+
37
+ name: str
38
+
39
+
40
+ T = TypeVar("T", bound=_Named)
41
+
42
+
43
+ def _resolve_by_name(items: Sequence[T] | None, name: str, kind: str) -> T:
44
+ """Find a config object by name.
45
+
46
+ Parameters
47
+ ----------
48
+ items : list[T] | None
49
+ List of config objects with a ``name`` attribute.
50
+ name : str
51
+ Name to look up.
52
+ kind : str
53
+ Human-readable kind for error messages (e.g. "dataset").
54
+
55
+ Raises
56
+ ------
57
+ ValueError
58
+ If *items* is ``None`` or *name* is not found.
59
+ """
60
+ if items is None:
61
+ raise ValueError(f"No {kind} configs defined, cannot resolve '{name}'")
62
+ for item in items:
63
+ if item.name == name:
64
+ return item
65
+ available = [item.name for item in items]
66
+ raise ValueError(f"Unknown {kind}: '{name}'. Available: {available}")
67
+
68
+
69
+ def _resolve_workflow(
70
+ workflow_name: str,
71
+ config: "PipelineConfig",
72
+ ) -> "WorkflowConfig":
73
+ """Resolve a workflow by name from ``config.workflows``."""
74
+ return _resolve_by_name(config.workflows, workflow_name, "workflow")
75
+
76
+
77
+ E = TypeVar("E", bound=BaseModel)
78
+
79
+
80
+ def _resolve_extractor_paths(extractor_cfg: E, data_dir: Path | None) -> E:
81
+ """Resolve relative ``model_path`` on extractor configs against *data_dir*."""
82
+ model_path: str | None = getattr(extractor_cfg, "model_path", None)
83
+
84
+ if model_path is not None:
85
+ from dataeval_flow.config._loader import resolve_path
86
+
87
+ resolved = str(resolve_path(model_path, data_dir))
88
+ if resolved != model_path:
89
+ return extractor_cfg.model_copy(update={"model_path": resolved})
90
+
91
+ return extractor_cfg
92
+
93
+
94
+ def _run_single_task(
95
+ task: "TaskConfig",
96
+ config: "PipelineConfig",
97
+ data_dir: Path | None = None,
98
+ cache_dir: Path | None = None,
99
+ ) -> "WorkflowResult[Any, Any]":
100
+ """Run a single resolved task against a pipeline config.
101
+
102
+ This is the internal workhorse — resolves all references (sources,
103
+ extractor) against ``PipelineConfig``, builds contexts, and executes
104
+ the workflow.
105
+ """
106
+ from dataeval_flow.cache import DatasetCache
107
+ from dataeval_flow.config._models import SourceConfig
108
+ from dataeval_flow.config.schemas import ExtractorConfig, PreprocessorConfig, SelectionConfig
109
+ from dataeval_flow.dataset import resolve_dataset
110
+ from dataeval_flow.preprocessing import build_preprocessing
111
+ from dataeval_flow.workflow import DatasetContext, WorkflowContext, get_workflow
112
+
113
+ logger.info("Task '%s': starting (workflow_instance=%s)", task.name, task.workflow)
114
+
115
+ # 1. Normalize sources to list
116
+ source_names: list[str] = [task.sources] if isinstance(task.sources, str) else list(task.sources)
117
+
118
+ # 2. Resolve extractor config (optional — single per task)
119
+ extractor_cfg: ExtractorConfig | None = None
120
+ transforms = None
121
+ batch_size: int | None = None
122
+
123
+ if task.extractor is not None:
124
+ extractor_cfg = _resolve_by_name(config.extractors, task.extractor, "extractor")
125
+ extractor_cfg = _resolve_extractor_paths(extractor_cfg, data_dir)
126
+ batch_size = extractor_cfg.batch_size
127
+
128
+ # Resolve preprocessor from extractor (optional)
129
+ if extractor_cfg.preprocessor is not None:
130
+ pre_config: PreprocessorConfig = _resolve_by_name(
131
+ config.preprocessors, extractor_cfg.preprocessor, "preprocessor"
132
+ )
133
+ transforms = build_preprocessing(pre_config.steps)
134
+
135
+ # 3. Build a DatasetContext per source
136
+ dataset_contexts: dict[str, DatasetContext] = {}
137
+ dataset_names: list[str] = []
138
+ resolved_sources: list[SourceConfig] = []
139
+
140
+ for src_name in source_names:
141
+ source: SourceConfig = _resolve_by_name(config.sources, src_name, "source")
142
+ resolved_sources.append(source)
143
+ ds_config = _resolve_by_name(config.datasets, source.dataset, "dataset")
144
+ resolved = resolve_dataset(ds_config, data_dir=data_dir)
145
+ dataset_names.append(source.dataset)
146
+
147
+ # Resolve selection from source (optional)
148
+ selection_steps = None
149
+ if source.selection is not None:
150
+ sel_config: SelectionConfig = _resolve_by_name(config.selections, source.selection, "selection")
151
+ selection_steps = sel_config.steps
152
+
153
+ # Build per-dataset cache
154
+ ds_cache = DatasetCache.get_or_create(
155
+ cache_dir=cache_dir,
156
+ name=resolved.name,
157
+ cache_key=resolved.cache_key,
158
+ )
159
+
160
+ dataset_contexts[src_name] = DatasetContext(
161
+ name=src_name,
162
+ dataset=resolved.dataset,
163
+ extractor=extractor_cfg,
164
+ transforms=transforms,
165
+ selection_steps=selection_steps,
166
+ batch_size=batch_size,
167
+ label_source=resolved.label_source,
168
+ cache=ds_cache,
169
+ )
170
+
171
+ if cache_dir:
172
+ logger.info("Cache enabled: %s", cache_dir)
173
+
174
+ # 4. Build WorkflowContext
175
+ context = WorkflowContext(
176
+ dataset_contexts=dataset_contexts,
177
+ batch_size=batch_size,
178
+ )
179
+
180
+ logger.debug("Task '%s': resolved %d source(s): %s", task.name, len(source_names), source_names)
181
+
182
+ # 5. Resolve workflow → type + params
183
+ instance = _resolve_workflow(task.workflow, config)
184
+ workflow = get_workflow(instance.type)
185
+
186
+ # 6. Run workflow with timing
187
+ logger.debug("Task '%s': executing workflow", task.name)
188
+ start = time.monotonic()
189
+ result = workflow.execute(context, instance)
190
+ elapsed = time.monotonic() - start
191
+ logger.info("Task '%s': finished in %.1fs (success=%s)", task.name, elapsed, result.success)
192
+
193
+ # 7. Populate metadata envelope
194
+ _populate_result_metadata(
195
+ result,
196
+ dataset_names,
197
+ dataset_contexts,
198
+ resolved_sources,
199
+ extractor_cfg,
200
+ elapsed,
201
+ instance,
202
+ config,
203
+ data_dir=data_dir,
204
+ )
205
+
206
+ return result
207
+
208
+
209
+ def _populate_result_metadata(
210
+ result: "WorkflowResult[Any, Any]",
211
+ dataset_names: Sequence[str],
212
+ dataset_contexts: "Mapping[str, DatasetContext]",
213
+ sources: "Sequence[SourceConfig]",
214
+ extractor_cfg: Any,
215
+ elapsed: float,
216
+ workflow_instance: "WorkflowConfig | None" = None,
217
+ pipeline_config: "PipelineConfig | None" = None,
218
+ data_dir: Path | None = None,
219
+ ) -> None:
220
+ """Fill in the JATIC metadata envelope from resolved source/extractor context."""
221
+ from dataeval_flow import __version__
222
+
223
+ result.metadata.dataset_id = dataset_names[0] if len(dataset_names) == 1 else ",".join(dataset_names)
224
+ result.metadata.tool_version = __version__
225
+ result.metadata.execution_time_s = round(elapsed, 3)
226
+
227
+ # Source context — selection info
228
+ selection_names = [s.selection for s in sources if s.selection is not None]
229
+ if selection_names:
230
+ result.metadata.selection_id = selection_names[0] if len(selection_names) == 1 else ",".join(selection_names)
231
+
232
+ # Build human-readable source descriptions: "src_name (dataset[selection])"
233
+ source_descs: list[str] = []
234
+ for src in sources:
235
+ if src.selection is not None:
236
+ source_descs.append(f"{src.name} ({src.dataset}[{src.selection}])")
237
+ else:
238
+ source_descs.append(f"{src.name} ({src.dataset})")
239
+ result.metadata.source_descriptions = source_descs
240
+
241
+ # Extractor context — model + preprocessor info
242
+ if extractor_cfg is not None:
243
+ result.metadata.model_id = f"{extractor_cfg.name} ({extractor_cfg.model})"
244
+ if extractor_cfg.preprocessor is not None:
245
+ result.metadata.preprocessor_id = extractor_cfg.preprocessor
246
+
247
+ # Annotate dataset source when label provenance is known
248
+ dc = next(iter(dataset_contexts.values()))
249
+ if dc.label_source:
250
+ result.metadata.label_source = dc.label_source
251
+
252
+ # Build fully resolved config snapshot for report traceability
253
+ result.metadata.resolved_config = _build_resolved_config(
254
+ sources, workflow_instance, extractor_cfg, pipeline_config, data_dir=data_dir
255
+ )
256
+
257
+
258
+ def _build_resolved_config(
259
+ sources: "Sequence[SourceConfig]",
260
+ workflow_instance: "WorkflowConfig | None",
261
+ extractor_cfg: Any,
262
+ pipeline_config: "PipelineConfig | None",
263
+ data_dir: Path | None = None,
264
+ ) -> dict[str, Any]:
265
+ """Build a fully resolved config dict for report traceability."""
266
+ cfg: dict[str, Any] = {}
267
+
268
+ # Sources — expand dataset and selection configs inline
269
+ source_entries: list[dict[str, Any]] = []
270
+ for src in sources:
271
+ entry: dict[str, Any] = {"name": src.name, "dataset": src.dataset}
272
+ if pipeline_config is not None:
273
+ ds = _resolve_by_name(pipeline_config.datasets, src.dataset, "dataset")
274
+ if getattr(ds, "serializable", True):
275
+ entry["dataset_config"] = ds.model_dump(mode="json")
276
+ else:
277
+ dumped = ds.model_dump(mode="json", exclude={"dataset"})
278
+ runtime_obj = getattr(ds, "dataset", None)
279
+ dumped["dataset"] = {
280
+ "type": "protocol",
281
+ "class": type(runtime_obj).__qualname__ if runtime_obj is not None else "unknown",
282
+ "id": getattr(runtime_obj, "metadata", {}).get("id", "unknown"),
283
+ }
284
+ entry["dataset_config"] = dumped
285
+ if src.selection is not None:
286
+ entry["selection"] = src.selection
287
+ if pipeline_config is not None:
288
+ sel = _resolve_by_name(pipeline_config.selections, src.selection, "selection")
289
+ entry["selection_config"] = sel.model_dump(mode="json")
290
+ source_entries.append(entry)
291
+ cfg["sources"] = source_entries
292
+
293
+ # Workflow params
294
+ if workflow_instance is not None:
295
+ cfg["workflow"] = workflow_instance.model_dump(mode="json")
296
+
297
+ # Extractor
298
+ if extractor_cfg is not None:
299
+ cfg["extractor"] = extractor_cfg.model_dump(mode="json")
300
+
301
+ return _relativize_paths(cfg, root=data_dir)
302
+
303
+
304
+ def _relativize_paths(obj: Any, root: Path | None = None) -> Any:
305
+ """Recursively convert absolute path strings to relative paths.
306
+
307
+ Only strings that resolve to a path under *root* are relativized.
308
+ If *root* is ``None``, the object is returned unchanged.
309
+ """
310
+ if root is None:
311
+ return obj
312
+ root = root.resolve()
313
+ if isinstance(obj, dict):
314
+ return {k: _relativize_paths(v, root) for k, v in obj.items()}
315
+ if isinstance(obj, list):
316
+ return [_relativize_paths(v, root) for v in obj]
317
+ if isinstance(obj, str) and obj.startswith("/"):
318
+ p = Path(obj)
319
+ try:
320
+ return str(p.relative_to(root))
321
+ except ValueError:
322
+ return obj
323
+ return obj
324
+
325
+
326
+ def run_tasks(
327
+ config: "PipelineConfig",
328
+ tasks: str | Sequence[str] | None = None,
329
+ data_dir: Path | None = None,
330
+ cache_dir: Path | None = None,
331
+ ) -> "list[WorkflowResult[Any, Any]]":
332
+ """Run tasks from a pipeline configuration.
333
+
334
+ Parameters
335
+ ----------
336
+ config : PipelineConfig
337
+ Pipeline configuration containing datasets, sources, extractors,
338
+ workflows, and tasks.
339
+ tasks : str | list[str] | None
340
+ Which tasks to run:
341
+
342
+ - ``None`` (default) — run all enabled tasks
343
+ - ``str`` — run a single task by name
344
+ - ``list[str]`` — run specific tasks by name, in the given order
345
+ data_dir : Path | None
346
+ Root directory for resolving relative paths in configs.
347
+ cache_dir : Path | None
348
+ Directory for disk-backed computation cache.
349
+
350
+ Returns
351
+ -------
352
+ list[WorkflowResult]
353
+ One result per task executed, in execution order.
354
+
355
+ Raises
356
+ ------
357
+ ValueError
358
+ If no tasks are defined, all are disabled, or a named task is
359
+ not found.
360
+ """
361
+ if not config.tasks:
362
+ raise ValueError("No tasks defined in pipeline config")
363
+
364
+ if tasks is None:
365
+ # Run all enabled tasks
366
+ to_run = [t for t in config.tasks if t.enabled]
367
+ skipped = len(config.tasks) - len(to_run)
368
+ if skipped:
369
+ logger.info("Skipping %d disabled task(s)", skipped)
370
+ if not to_run:
371
+ raise ValueError("All tasks are disabled — nothing to run")
372
+ elif isinstance(tasks, str):
373
+ to_run = [_resolve_by_name(config.tasks, tasks, "task")]
374
+ else:
375
+ to_run = [_resolve_by_name(config.tasks, name, "task") for name in tasks]
376
+
377
+ logger.info("Running %d task(s)", len(to_run))
378
+ results: list[WorkflowResult[Any, Any]] = []
379
+ for task in to_run:
380
+ logger.info("--- Task: %s (workflow: %s) ---", task.name, task.workflow)
381
+ results.append(_run_single_task(task, config, data_dir=data_dir, cache_dir=cache_dir))
382
+ return results
383
+
384
+
385
+ @overload
386
+ def run_task(
387
+ task: "DataAnalysisTaskConfig",
388
+ config: "PipelineConfig",
389
+ data_dir: Path | None = None,
390
+ cache_dir: Path | None = None,
391
+ ) -> "WorkflowResult[DataAnalysisMetadata, DataAnalysisOutputs]": ...
392
+ @overload
393
+ def run_task(
394
+ task: "DataCleaningTaskConfig",
395
+ config: "PipelineConfig",
396
+ data_dir: Path | None = None,
397
+ cache_dir: Path | None = None,
398
+ ) -> "WorkflowResult[DataCleaningMetadata, DataCleaningOutputs]": ...
399
+ @overload
400
+ def run_task(
401
+ task: "DriftMonitoringTaskConfig",
402
+ config: "PipelineConfig",
403
+ data_dir: Path | None = None,
404
+ cache_dir: Path | None = None,
405
+ ) -> "WorkflowResult[DriftMonitoringMetadata, DriftMonitoringOutputs]": ...
406
+ @overload
407
+ def run_task(
408
+ task: "OODDetectionTaskConfig",
409
+ config: "PipelineConfig",
410
+ data_dir: Path | None = None,
411
+ cache_dir: Path | None = None,
412
+ ) -> "WorkflowResult[OODDetectionMetadata, OODDetectionOutputs]": ...
413
+ @overload
414
+ def run_task(
415
+ task: "DataPrioritizationTaskConfig",
416
+ config: "PipelineConfig",
417
+ data_dir: Path | None = None,
418
+ cache_dir: Path | None = None,
419
+ ) -> "WorkflowResult[DataPrioritizationMetadata, DataPrioritizationOutputs]": ...
420
+ @overload
421
+ def run_task(
422
+ task: "TaskConfig", config: "PipelineConfig", data_dir: Path | None = None, cache_dir: Path | None = None
423
+ ) -> "WorkflowResult[Any, Any]": ...
424
+ def run_task(
425
+ task: "TaskConfig", config: "PipelineConfig", data_dir: Path | None = None, cache_dir: Path | None = None
426
+ ) -> "WorkflowResult[Any, Any]":
427
+ """Run a single task, returning a narrowly typed result based on the task type.
428
+
429
+ Unlike :func:`run_tasks`, this function accepts the task config object
430
+ directly rather than looking it up by name, which allows type checkers to
431
+ narrow the return type to the appropriate workflow result type.
432
+
433
+ Parameters
434
+ ----------
435
+ task : TaskConfig
436
+ The task configuration to execute.
437
+ config : PipelineConfig
438
+ Pipeline configuration supplying datasets, sources, extractors, and
439
+ workflow definitions. The task does **not** need to appear in
440
+ ``config.tasks``.
441
+ data_dir : Path | None
442
+ Root directory for resolving relative paths in configs.
443
+ cache_dir : Path | None
444
+ Directory for disk-backed computation cache.
445
+
446
+ Returns
447
+ -------
448
+ WorkflowResult
449
+ A result typed to the specific workflow — e.g.
450
+ ``WorkflowResult[OODDetectionMetadata, OODDetectionOutputs]`` when
451
+ *task* is an :class:`~dataeval_flow.config.OODDetectionTaskConfig`.
452
+ """
453
+ logger.info("--- Task: %s (workflow: %s) ---", task.name, task.workflow)
454
+ return _run_single_task(task, config, data_dir=data_dir, cache_dir=cache_dir)
@@ -0,0 +1 @@
1
+ """Concrete workflow implementations."""
@@ -0,0 +1,38 @@
1
+ """Data analysis workflow - public re-exports."""
2
+
3
+ from dataeval_flow.workflows.analysis.outputs import (
4
+ BiasResult,
5
+ CrossSplitLabelHealth,
6
+ CrossSplitRedundancy,
7
+ DataAnalysisMetadata,
8
+ DataAnalysisOutputs,
9
+ DataAnalysisRawOutputs,
10
+ DataAnalysisReport,
11
+ DataAnalysisResult,
12
+ DistributionShiftResult,
13
+ ImageQualityResult,
14
+ LabelHealthResult,
15
+ RedundancyResult,
16
+ is_analysis_result,
17
+ )
18
+ from dataeval_flow.workflows.analysis.params import DataAnalysisHealthThresholds, DataAnalysisParameters
19
+ from dataeval_flow.workflows.analysis.workflow import DataAnalysisWorkflow
20
+
21
+ __all__ = [
22
+ "BiasResult",
23
+ "CrossSplitLabelHealth",
24
+ "CrossSplitRedundancy",
25
+ "DataAnalysisHealthThresholds",
26
+ "DataAnalysisMetadata",
27
+ "DataAnalysisOutputs",
28
+ "DataAnalysisParameters",
29
+ "DataAnalysisRawOutputs",
30
+ "DataAnalysisReport",
31
+ "DataAnalysisResult",
32
+ "DataAnalysisWorkflow",
33
+ "DistributionShiftResult",
34
+ "ImageQualityResult",
35
+ "LabelHealthResult",
36
+ "RedundancyResult",
37
+ "is_analysis_result",
38
+ ]
@@ -0,0 +1,202 @@
1
+ """Data analysis workflow outputs."""
2
+
3
+ from typing import TYPE_CHECKING, Any, Literal, TypeAlias
4
+
5
+ from pydantic import BaseModel, Field
6
+ from typing_extensions import TypeIs
7
+
8
+ from dataeval_flow.config.schemas import ResultMetadata
9
+ from dataeval_flow.workflow.base import Reportable, WorkflowOutputsBase, WorkflowReportBase
10
+
11
+ if TYPE_CHECKING:
12
+ from dataeval_flow.workflow import WorkflowResult
13
+
14
+ __all__ = [
15
+ "BiasResult",
16
+ "CrossSplitLabelHealth",
17
+ "CrossSplitRedundancy",
18
+ "CrossSplitResult",
19
+ "DataAnalysisMetadata",
20
+ "DataAnalysisOutputs",
21
+ "DataAnalysisRawOutputs",
22
+ "DataAnalysisReport",
23
+ "DataAnalysisResult",
24
+ "DistributionShiftResult",
25
+ "ImageQualityResult",
26
+ "LabelHealthResult",
27
+ "RedundancyResult",
28
+ "SplitResult",
29
+ "is_analysis_result",
30
+ ]
31
+
32
+
33
+ # ---------------------------------------------------------------------------
34
+ # Per-split assessment sub-models
35
+ # ---------------------------------------------------------------------------
36
+
37
+
38
+ class ImageQualityResult(BaseModel):
39
+ """Image-level anomaly detection results."""
40
+
41
+ outlier_count: int = Field(description="Number of items flagged as outliers")
42
+ outlier_rate: float = Field(description="Fraction of items flagged as outliers")
43
+ outlier_summary: dict[str, int] = Field(description="Count of outlier items per metric name")
44
+
45
+
46
+ class RedundancyResult(BaseModel):
47
+ """Duplicate and near-duplicate detection results."""
48
+
49
+ exact_duplicate_groups: int = Field(description="Number of exact duplicate groups found")
50
+ near_duplicate_groups: int = Field(description="Number of near duplicate groups found")
51
+ exact_duplicates_count: int = Field(description="Total items across all exact duplicate groups")
52
+ near_duplicates_count: int = Field(description="Total items across all near duplicate groups")
53
+ exact_groups: list[list[int]] = Field(default_factory=list, description="Indices per exact duplicate group")
54
+ near_groups: list[list[int]] = Field(default_factory=list, description="Indices per near duplicate group")
55
+
56
+
57
+ class LabelHealthResult(BaseModel):
58
+ """Label completeness and distribution results."""
59
+
60
+ num_classes: int = Field(description="Number of unique class labels")
61
+ class_distribution: dict[str, int] = Field(description="Mapping of class name to label count")
62
+ empty_images: list[int] = Field(description="Indices of images with no labels")
63
+
64
+
65
+ class BiasResult(BaseModel):
66
+ """Metadata bias analysis results."""
67
+
68
+ metadata_factors: list[str] = Field(description="Names of metadata factors present")
69
+ metadata_summary: dict[str, dict[str, Any]] = Field(description="Per-factor summary statistics")
70
+ balance_summary: dict[str, Any] | None = Field(default=None, description="Balance (MI) analysis results")
71
+ diversity_summary: dict[str, Any] | None = Field(default=None, description="Diversity analysis results")
72
+
73
+
74
+ # ---------------------------------------------------------------------------
75
+ # Per-split result (composed from assessment sub-models)
76
+ # ---------------------------------------------------------------------------
77
+
78
+
79
+ class SplitResult(BaseModel):
80
+ """Per-split summary statistics grouped by assessment area."""
81
+
82
+ num_samples: int = Field(description="Total number of images in the split")
83
+ image_quality: ImageQualityResult = Field(description="Image-level anomaly detection")
84
+ redundancy: RedundancyResult = Field(description="Duplicate detection")
85
+ label_health: LabelHealthResult = Field(description="Label completeness and distribution")
86
+ bias: BiasResult = Field(description="Metadata bias analysis")
87
+
88
+
89
+ # ---------------------------------------------------------------------------
90
+ # Cross-split assessment sub-models
91
+ # ---------------------------------------------------------------------------
92
+
93
+
94
+ class CrossSplitRedundancy(BaseModel):
95
+ """Cross-split duplicate leakage detection."""
96
+
97
+ duplicate_leakage: dict[str, Any] = Field(
98
+ default_factory=lambda: {"exact_count": 0, "near_count": 0, "exact_groups": [], "near_groups": []},
99
+ description="Cross-split duplicate detection results",
100
+ )
101
+
102
+
103
+ class CrossSplitLabelHealth(BaseModel):
104
+ """Cross-split label comparison results."""
105
+
106
+ label_overlap: dict[str, Any] = Field(
107
+ description="Class-level comparison including shared classes and proportion differences"
108
+ )
109
+ label_parity: dict[str, Any] | None = Field(
110
+ default=None,
111
+ description="Chi-squared label distribution parity test between splits",
112
+ )
113
+
114
+
115
+ class DistributionShiftResult(BaseModel):
116
+ """Cross-split embedding divergence results."""
117
+
118
+ divergence: float | None = Field(
119
+ default=None,
120
+ description="Embedding-space divergence between splits, or None if no extractor was provided",
121
+ )
122
+ divergence_method: str | None = Field(
123
+ default=None,
124
+ description='Method used for divergence computation ("mst" or "fnn")',
125
+ )
126
+
127
+
128
+ # ---------------------------------------------------------------------------
129
+ # Cross-split result (composed from assessment sub-models)
130
+ # ---------------------------------------------------------------------------
131
+
132
+
133
+ class CrossSplitResult(BaseModel):
134
+ """Comparison results between two dataset splits."""
135
+
136
+ redundancy: CrossSplitRedundancy = Field(description="Cross-split duplicate leakage")
137
+ label_health: CrossSplitLabelHealth = Field(description="Label distribution comparison")
138
+ distribution_shift: DistributionShiftResult = Field(description="Embedding divergence")
139
+
140
+
141
+ # ---------------------------------------------------------------------------
142
+ # Workflow output models
143
+ # ---------------------------------------------------------------------------
144
+
145
+
146
+ class DataAnalysisRawOutputs(WorkflowOutputsBase):
147
+ """Machine-readable results from data analysis workflow.
148
+
149
+ ``dataset_size`` is the total number of items across all splits.
150
+ """
151
+
152
+ splits: dict[str, SplitResult] = Field(
153
+ default_factory=dict,
154
+ description="Per-split quality summaries keyed by split name",
155
+ )
156
+ cross_split: dict[str, CrossSplitResult] = Field(
157
+ default_factory=dict,
158
+ description='Pairwise cross-split comparisons keyed by "splitA_vs_splitB"',
159
+ )
160
+
161
+
162
+ class DataAnalysisReport(WorkflowReportBase):
163
+ """Human-readable report for data analysis workflow."""
164
+
165
+ findings: list[Reportable] = Field(default_factory=list)
166
+
167
+
168
+ class DataAnalysisOutputs(BaseModel):
169
+ """Complete data analysis workflow output."""
170
+
171
+ raw: DataAnalysisRawOutputs
172
+ report: DataAnalysisReport
173
+
174
+
175
+ class DataAnalysisMetadata(ResultMetadata):
176
+ """Metadata for the data-analysis workflow."""
177
+
178
+ mode: Literal["advisory", "preparatory"] = "advisory"
179
+ split_names: list[str] = Field(default_factory=list)
180
+
181
+
182
+ # ---------------------------------------------------------------------------
183
+ # Type alias and TypeIs guard for type narrowing
184
+ # ---------------------------------------------------------------------------
185
+
186
+ #: Fully typed result alias for the data-analysis workflow.
187
+ DataAnalysisResult: TypeAlias = "WorkflowResult[DataAnalysisMetadata, DataAnalysisOutputs]"
188
+
189
+
190
+ def is_analysis_result(
191
+ result: "WorkflowResult[Any, Any]",
192
+ ) -> TypeIs["WorkflowResult[DataAnalysisMetadata, DataAnalysisOutputs]"]:
193
+ """Narrow a generic ``WorkflowResult`` to a data-analysis result.
194
+
195
+ Useful in the CLI loop or any code that receives a generic result::
196
+
197
+ [result] = run_tasks(config, "my_task")
198
+ if is_analysis_result(result):
199
+ result.metadata.split_names # ✓ typed
200
+ result.data.raw.splits # ✓ typed
201
+ """
202
+ return isinstance(result.metadata, DataAnalysisMetadata)