dataeval-flow 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataeval_flow/__init__.py +93 -0
- dataeval_flow/__main__.py +149 -0
- dataeval_flow/_app/__init__.py +5 -0
- dataeval_flow/_app/_model/__init__.py +5 -0
- dataeval_flow/_app/_model/_coerce.py +126 -0
- dataeval_flow/_app/_model/_discover.py +171 -0
- dataeval_flow/_app/_model/_execution.py +108 -0
- dataeval_flow/_app/_model/_introspect.py +280 -0
- dataeval_flow/_app/_model/_item.py +213 -0
- dataeval_flow/_app/_model/_registry.py +255 -0
- dataeval_flow/_app/_model/_state.py +322 -0
- dataeval_flow/_app/_model/_undo.py +61 -0
- dataeval_flow/_app/_panes/__init__.py +35 -0
- dataeval_flow/_app/_panes/_config_pane.py +173 -0
- dataeval_flow/_app/_panes/_result_pane.py +125 -0
- dataeval_flow/_app/_panes/_task_pane.py +91 -0
- dataeval_flow/_app/_panes/_widgets.py +111 -0
- dataeval_flow/_app/_screens/__init__.py +25 -0
- dataeval_flow/_app/_screens/_base.py +242 -0
- dataeval_flow/_app/_screens/_detail.py +333 -0
- dataeval_flow/_app/_screens/_model.py +102 -0
- dataeval_flow/_app/_screens/_params.py +80 -0
- dataeval_flow/_app/_screens/_pathpicker.py +68 -0
- dataeval_flow/_app/_screens/_section.py +621 -0
- dataeval_flow/_app/_screens/_settings.py +183 -0
- dataeval_flow/_app/_viewmodel/__init__.py +15 -0
- dataeval_flow/_app/_viewmodel/_builder_vm.py +272 -0
- dataeval_flow/_app/_viewmodel/_model_vm.py +70 -0
- dataeval_flow/_app/_viewmodel/_rendering.py +189 -0
- dataeval_flow/_app/_viewmodel/_result_vm.py +210 -0
- dataeval_flow/_app/_viewmodel/_section_vm.py +224 -0
- dataeval_flow/_app/app.py +742 -0
- dataeval_flow/_app/cli.py +592 -0
- dataeval_flow/_logging.py +102 -0
- dataeval_flow/cache.py +1355 -0
- dataeval_flow/config/__init__.py +80 -0
- dataeval_flow/config/_loader.py +79 -0
- dataeval_flow/config/_merge.py +92 -0
- dataeval_flow/config/_models.py +115 -0
- dataeval_flow/config/_paths.py +85 -0
- dataeval_flow/config/schemas/__init__.py +112 -0
- dataeval_flow/config/schemas/_dataset.py +111 -0
- dataeval_flow/config/schemas/_extractor.py +119 -0
- dataeval_flow/config/schemas/_metadata.py +28 -0
- dataeval_flow/config/schemas/_preprocessor.py +18 -0
- dataeval_flow/config/schemas/_selection.py +100 -0
- dataeval_flow/config/schemas/_task.py +89 -0
- dataeval_flow/config/schemas/_workflow.py +135 -0
- dataeval_flow/dataset.py +635 -0
- dataeval_flow/embeddings.py +135 -0
- dataeval_flow/metadata.py +48 -0
- dataeval_flow/preprocessing.py +141 -0
- dataeval_flow/py.typed +0 -0
- dataeval_flow/runner.py +118 -0
- dataeval_flow/selection.py +50 -0
- dataeval_flow/workflow/__init__.py +328 -0
- dataeval_flow/workflow/_text_report.py +511 -0
- dataeval_flow/workflow/base.py +69 -0
- dataeval_flow/workflow/orchestrator.py +454 -0
- dataeval_flow/workflows/__init__.py +1 -0
- dataeval_flow/workflows/analysis/__init__.py +38 -0
- dataeval_flow/workflows/analysis/outputs.py +202 -0
- dataeval_flow/workflows/analysis/params.py +114 -0
- dataeval_flow/workflows/analysis/workflow.py +1313 -0
- dataeval_flow/workflows/cleaning/__init__.py +23 -0
- dataeval_flow/workflows/cleaning/outputs.py +200 -0
- dataeval_flow/workflows/cleaning/params.py +160 -0
- dataeval_flow/workflows/cleaning/report.py +304 -0
- dataeval_flow/workflows/cleaning/workflow.py +794 -0
- dataeval_flow/workflows/drift/__init__.py +1 -0
- dataeval_flow/workflows/drift/outputs.py +144 -0
- dataeval_flow/workflows/drift/params.py +332 -0
- dataeval_flow/workflows/drift/report.py +201 -0
- dataeval_flow/workflows/drift/workflow.py +647 -0
- dataeval_flow/workflows/ood/__init__.py +1 -0
- dataeval_flow/workflows/ood/outputs.py +134 -0
- dataeval_flow/workflows/ood/params.py +161 -0
- dataeval_flow/workflows/ood/report.py +311 -0
- dataeval_flow/workflows/ood/workflow.py +728 -0
- dataeval_flow/workflows/prioritization/__init__.py +1 -0
- dataeval_flow/workflows/prioritization/outputs.py +122 -0
- dataeval_flow/workflows/prioritization/params.py +124 -0
- dataeval_flow/workflows/prioritization/report.py +117 -0
- dataeval_flow/workflows/prioritization/workflow.py +587 -0
- dataeval_flow/workflows/splitting/__init__.py +25 -0
- dataeval_flow/workflows/splitting/outputs.py +101 -0
- dataeval_flow/workflows/splitting/params.py +61 -0
- dataeval_flow/workflows/splitting/report.py +485 -0
- dataeval_flow/workflows/splitting/workflow.py +371 -0
- dataeval_flow-0.1.0.dist-info/METADATA +305 -0
- dataeval_flow-0.1.0.dist-info/RECORD +94 -0
- dataeval_flow-0.1.0.dist-info/WHEEL +4 -0
- dataeval_flow-0.1.0.dist-info/entry_points.txt +2 -0
- dataeval_flow-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
"""Task orchestration — config → execution bridge."""
|
|
2
|
+
|
|
3
|
+
__all__ = ["run_task", "run_tasks"]
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Mapping, Sequence
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import TYPE_CHECKING, Any, Protocol, TypeVar, overload, runtime_checkable
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel
|
|
12
|
+
|
|
13
|
+
logger: logging.Logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from dataeval_flow.config import PipelineConfig, SourceConfig, TaskConfig
|
|
17
|
+
from dataeval_flow.config.schemas import WorkflowConfig
|
|
18
|
+
from dataeval_flow.config.schemas._task import (
|
|
19
|
+
DataAnalysisTaskConfig,
|
|
20
|
+
DataCleaningTaskConfig,
|
|
21
|
+
DataPrioritizationTaskConfig,
|
|
22
|
+
DriftMonitoringTaskConfig,
|
|
23
|
+
OODDetectionTaskConfig,
|
|
24
|
+
)
|
|
25
|
+
from dataeval_flow.workflow import DatasetContext, WorkflowResult
|
|
26
|
+
from dataeval_flow.workflows.analysis.outputs import DataAnalysisMetadata, DataAnalysisOutputs
|
|
27
|
+
from dataeval_flow.workflows.cleaning.outputs import DataCleaningMetadata, DataCleaningOutputs
|
|
28
|
+
from dataeval_flow.workflows.drift.outputs import DriftMonitoringMetadata, DriftMonitoringOutputs
|
|
29
|
+
from dataeval_flow.workflows.ood.outputs import OODDetectionMetadata, OODDetectionOutputs
|
|
30
|
+
from dataeval_flow.workflows.prioritization.outputs import DataPrioritizationMetadata, DataPrioritizationOutputs
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@runtime_checkable
|
|
34
|
+
class _Named(Protocol):
|
|
35
|
+
"""Protocol for config objects with a name attribute."""
|
|
36
|
+
|
|
37
|
+
name: str
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
T = TypeVar("T", bound=_Named)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _resolve_by_name(items: Sequence[T] | None, name: str, kind: str) -> T:
|
|
44
|
+
"""Find a config object by name.
|
|
45
|
+
|
|
46
|
+
Parameters
|
|
47
|
+
----------
|
|
48
|
+
items : list[T] | None
|
|
49
|
+
List of config objects with a ``name`` attribute.
|
|
50
|
+
name : str
|
|
51
|
+
Name to look up.
|
|
52
|
+
kind : str
|
|
53
|
+
Human-readable kind for error messages (e.g. "dataset").
|
|
54
|
+
|
|
55
|
+
Raises
|
|
56
|
+
------
|
|
57
|
+
ValueError
|
|
58
|
+
If *items* is ``None`` or *name* is not found.
|
|
59
|
+
"""
|
|
60
|
+
if items is None:
|
|
61
|
+
raise ValueError(f"No {kind} configs defined, cannot resolve '{name}'")
|
|
62
|
+
for item in items:
|
|
63
|
+
if item.name == name:
|
|
64
|
+
return item
|
|
65
|
+
available = [item.name for item in items]
|
|
66
|
+
raise ValueError(f"Unknown {kind}: '{name}'. Available: {available}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _resolve_workflow(
|
|
70
|
+
workflow_name: str,
|
|
71
|
+
config: "PipelineConfig",
|
|
72
|
+
) -> "WorkflowConfig":
|
|
73
|
+
"""Resolve a workflow by name from ``config.workflows``."""
|
|
74
|
+
return _resolve_by_name(config.workflows, workflow_name, "workflow")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
E = TypeVar("E", bound=BaseModel)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _resolve_extractor_paths(extractor_cfg: E, data_dir: Path | None) -> E:
|
|
81
|
+
"""Resolve relative ``model_path`` on extractor configs against *data_dir*."""
|
|
82
|
+
model_path: str | None = getattr(extractor_cfg, "model_path", None)
|
|
83
|
+
|
|
84
|
+
if model_path is not None:
|
|
85
|
+
from dataeval_flow.config._loader import resolve_path
|
|
86
|
+
|
|
87
|
+
resolved = str(resolve_path(model_path, data_dir))
|
|
88
|
+
if resolved != model_path:
|
|
89
|
+
return extractor_cfg.model_copy(update={"model_path": resolved})
|
|
90
|
+
|
|
91
|
+
return extractor_cfg
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _run_single_task(
|
|
95
|
+
task: "TaskConfig",
|
|
96
|
+
config: "PipelineConfig",
|
|
97
|
+
data_dir: Path | None = None,
|
|
98
|
+
cache_dir: Path | None = None,
|
|
99
|
+
) -> "WorkflowResult[Any, Any]":
|
|
100
|
+
"""Run a single resolved task against a pipeline config.
|
|
101
|
+
|
|
102
|
+
This is the internal workhorse — resolves all references (sources,
|
|
103
|
+
extractor) against ``PipelineConfig``, builds contexts, and executes
|
|
104
|
+
the workflow.
|
|
105
|
+
"""
|
|
106
|
+
from dataeval_flow.cache import DatasetCache
|
|
107
|
+
from dataeval_flow.config._models import SourceConfig
|
|
108
|
+
from dataeval_flow.config.schemas import ExtractorConfig, PreprocessorConfig, SelectionConfig
|
|
109
|
+
from dataeval_flow.dataset import resolve_dataset
|
|
110
|
+
from dataeval_flow.preprocessing import build_preprocessing
|
|
111
|
+
from dataeval_flow.workflow import DatasetContext, WorkflowContext, get_workflow
|
|
112
|
+
|
|
113
|
+
logger.info("Task '%s': starting (workflow_instance=%s)", task.name, task.workflow)
|
|
114
|
+
|
|
115
|
+
# 1. Normalize sources to list
|
|
116
|
+
source_names: list[str] = [task.sources] if isinstance(task.sources, str) else list(task.sources)
|
|
117
|
+
|
|
118
|
+
# 2. Resolve extractor config (optional — single per task)
|
|
119
|
+
extractor_cfg: ExtractorConfig | None = None
|
|
120
|
+
transforms = None
|
|
121
|
+
batch_size: int | None = None
|
|
122
|
+
|
|
123
|
+
if task.extractor is not None:
|
|
124
|
+
extractor_cfg = _resolve_by_name(config.extractors, task.extractor, "extractor")
|
|
125
|
+
extractor_cfg = _resolve_extractor_paths(extractor_cfg, data_dir)
|
|
126
|
+
batch_size = extractor_cfg.batch_size
|
|
127
|
+
|
|
128
|
+
# Resolve preprocessor from extractor (optional)
|
|
129
|
+
if extractor_cfg.preprocessor is not None:
|
|
130
|
+
pre_config: PreprocessorConfig = _resolve_by_name(
|
|
131
|
+
config.preprocessors, extractor_cfg.preprocessor, "preprocessor"
|
|
132
|
+
)
|
|
133
|
+
transforms = build_preprocessing(pre_config.steps)
|
|
134
|
+
|
|
135
|
+
# 3. Build a DatasetContext per source
|
|
136
|
+
dataset_contexts: dict[str, DatasetContext] = {}
|
|
137
|
+
dataset_names: list[str] = []
|
|
138
|
+
resolved_sources: list[SourceConfig] = []
|
|
139
|
+
|
|
140
|
+
for src_name in source_names:
|
|
141
|
+
source: SourceConfig = _resolve_by_name(config.sources, src_name, "source")
|
|
142
|
+
resolved_sources.append(source)
|
|
143
|
+
ds_config = _resolve_by_name(config.datasets, source.dataset, "dataset")
|
|
144
|
+
resolved = resolve_dataset(ds_config, data_dir=data_dir)
|
|
145
|
+
dataset_names.append(source.dataset)
|
|
146
|
+
|
|
147
|
+
# Resolve selection from source (optional)
|
|
148
|
+
selection_steps = None
|
|
149
|
+
if source.selection is not None:
|
|
150
|
+
sel_config: SelectionConfig = _resolve_by_name(config.selections, source.selection, "selection")
|
|
151
|
+
selection_steps = sel_config.steps
|
|
152
|
+
|
|
153
|
+
# Build per-dataset cache
|
|
154
|
+
ds_cache = DatasetCache.get_or_create(
|
|
155
|
+
cache_dir=cache_dir,
|
|
156
|
+
name=resolved.name,
|
|
157
|
+
cache_key=resolved.cache_key,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
dataset_contexts[src_name] = DatasetContext(
|
|
161
|
+
name=src_name,
|
|
162
|
+
dataset=resolved.dataset,
|
|
163
|
+
extractor=extractor_cfg,
|
|
164
|
+
transforms=transforms,
|
|
165
|
+
selection_steps=selection_steps,
|
|
166
|
+
batch_size=batch_size,
|
|
167
|
+
label_source=resolved.label_source,
|
|
168
|
+
cache=ds_cache,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
if cache_dir:
|
|
172
|
+
logger.info("Cache enabled: %s", cache_dir)
|
|
173
|
+
|
|
174
|
+
# 4. Build WorkflowContext
|
|
175
|
+
context = WorkflowContext(
|
|
176
|
+
dataset_contexts=dataset_contexts,
|
|
177
|
+
batch_size=batch_size,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
logger.debug("Task '%s': resolved %d source(s): %s", task.name, len(source_names), source_names)
|
|
181
|
+
|
|
182
|
+
# 5. Resolve workflow → type + params
|
|
183
|
+
instance = _resolve_workflow(task.workflow, config)
|
|
184
|
+
workflow = get_workflow(instance.type)
|
|
185
|
+
|
|
186
|
+
# 6. Run workflow with timing
|
|
187
|
+
logger.debug("Task '%s': executing workflow", task.name)
|
|
188
|
+
start = time.monotonic()
|
|
189
|
+
result = workflow.execute(context, instance)
|
|
190
|
+
elapsed = time.monotonic() - start
|
|
191
|
+
logger.info("Task '%s': finished in %.1fs (success=%s)", task.name, elapsed, result.success)
|
|
192
|
+
|
|
193
|
+
# 7. Populate metadata envelope
|
|
194
|
+
_populate_result_metadata(
|
|
195
|
+
result,
|
|
196
|
+
dataset_names,
|
|
197
|
+
dataset_contexts,
|
|
198
|
+
resolved_sources,
|
|
199
|
+
extractor_cfg,
|
|
200
|
+
elapsed,
|
|
201
|
+
instance,
|
|
202
|
+
config,
|
|
203
|
+
data_dir=data_dir,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
return result
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _populate_result_metadata(
|
|
210
|
+
result: "WorkflowResult[Any, Any]",
|
|
211
|
+
dataset_names: Sequence[str],
|
|
212
|
+
dataset_contexts: "Mapping[str, DatasetContext]",
|
|
213
|
+
sources: "Sequence[SourceConfig]",
|
|
214
|
+
extractor_cfg: Any,
|
|
215
|
+
elapsed: float,
|
|
216
|
+
workflow_instance: "WorkflowConfig | None" = None,
|
|
217
|
+
pipeline_config: "PipelineConfig | None" = None,
|
|
218
|
+
data_dir: Path | None = None,
|
|
219
|
+
) -> None:
|
|
220
|
+
"""Fill in the JATIC metadata envelope from resolved source/extractor context."""
|
|
221
|
+
from dataeval_flow import __version__
|
|
222
|
+
|
|
223
|
+
result.metadata.dataset_id = dataset_names[0] if len(dataset_names) == 1 else ",".join(dataset_names)
|
|
224
|
+
result.metadata.tool_version = __version__
|
|
225
|
+
result.metadata.execution_time_s = round(elapsed, 3)
|
|
226
|
+
|
|
227
|
+
# Source context — selection info
|
|
228
|
+
selection_names = [s.selection for s in sources if s.selection is not None]
|
|
229
|
+
if selection_names:
|
|
230
|
+
result.metadata.selection_id = selection_names[0] if len(selection_names) == 1 else ",".join(selection_names)
|
|
231
|
+
|
|
232
|
+
# Build human-readable source descriptions: "src_name (dataset[selection])"
|
|
233
|
+
source_descs: list[str] = []
|
|
234
|
+
for src in sources:
|
|
235
|
+
if src.selection is not None:
|
|
236
|
+
source_descs.append(f"{src.name} ({src.dataset}[{src.selection}])")
|
|
237
|
+
else:
|
|
238
|
+
source_descs.append(f"{src.name} ({src.dataset})")
|
|
239
|
+
result.metadata.source_descriptions = source_descs
|
|
240
|
+
|
|
241
|
+
# Extractor context — model + preprocessor info
|
|
242
|
+
if extractor_cfg is not None:
|
|
243
|
+
result.metadata.model_id = f"{extractor_cfg.name} ({extractor_cfg.model})"
|
|
244
|
+
if extractor_cfg.preprocessor is not None:
|
|
245
|
+
result.metadata.preprocessor_id = extractor_cfg.preprocessor
|
|
246
|
+
|
|
247
|
+
# Annotate dataset source when label provenance is known
|
|
248
|
+
dc = next(iter(dataset_contexts.values()))
|
|
249
|
+
if dc.label_source:
|
|
250
|
+
result.metadata.label_source = dc.label_source
|
|
251
|
+
|
|
252
|
+
# Build fully resolved config snapshot for report traceability
|
|
253
|
+
result.metadata.resolved_config = _build_resolved_config(
|
|
254
|
+
sources, workflow_instance, extractor_cfg, pipeline_config, data_dir=data_dir
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _build_resolved_config(
|
|
259
|
+
sources: "Sequence[SourceConfig]",
|
|
260
|
+
workflow_instance: "WorkflowConfig | None",
|
|
261
|
+
extractor_cfg: Any,
|
|
262
|
+
pipeline_config: "PipelineConfig | None",
|
|
263
|
+
data_dir: Path | None = None,
|
|
264
|
+
) -> dict[str, Any]:
|
|
265
|
+
"""Build a fully resolved config dict for report traceability."""
|
|
266
|
+
cfg: dict[str, Any] = {}
|
|
267
|
+
|
|
268
|
+
# Sources — expand dataset and selection configs inline
|
|
269
|
+
source_entries: list[dict[str, Any]] = []
|
|
270
|
+
for src in sources:
|
|
271
|
+
entry: dict[str, Any] = {"name": src.name, "dataset": src.dataset}
|
|
272
|
+
if pipeline_config is not None:
|
|
273
|
+
ds = _resolve_by_name(pipeline_config.datasets, src.dataset, "dataset")
|
|
274
|
+
if getattr(ds, "serializable", True):
|
|
275
|
+
entry["dataset_config"] = ds.model_dump(mode="json")
|
|
276
|
+
else:
|
|
277
|
+
dumped = ds.model_dump(mode="json", exclude={"dataset"})
|
|
278
|
+
runtime_obj = getattr(ds, "dataset", None)
|
|
279
|
+
dumped["dataset"] = {
|
|
280
|
+
"type": "protocol",
|
|
281
|
+
"class": type(runtime_obj).__qualname__ if runtime_obj is not None else "unknown",
|
|
282
|
+
"id": getattr(runtime_obj, "metadata", {}).get("id", "unknown"),
|
|
283
|
+
}
|
|
284
|
+
entry["dataset_config"] = dumped
|
|
285
|
+
if src.selection is not None:
|
|
286
|
+
entry["selection"] = src.selection
|
|
287
|
+
if pipeline_config is not None:
|
|
288
|
+
sel = _resolve_by_name(pipeline_config.selections, src.selection, "selection")
|
|
289
|
+
entry["selection_config"] = sel.model_dump(mode="json")
|
|
290
|
+
source_entries.append(entry)
|
|
291
|
+
cfg["sources"] = source_entries
|
|
292
|
+
|
|
293
|
+
# Workflow params
|
|
294
|
+
if workflow_instance is not None:
|
|
295
|
+
cfg["workflow"] = workflow_instance.model_dump(mode="json")
|
|
296
|
+
|
|
297
|
+
# Extractor
|
|
298
|
+
if extractor_cfg is not None:
|
|
299
|
+
cfg["extractor"] = extractor_cfg.model_dump(mode="json")
|
|
300
|
+
|
|
301
|
+
return _relativize_paths(cfg, root=data_dir)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _relativize_paths(obj: Any, root: Path | None = None) -> Any:
|
|
305
|
+
"""Recursively convert absolute path strings to relative paths.
|
|
306
|
+
|
|
307
|
+
Only strings that resolve to a path under *root* are relativized.
|
|
308
|
+
If *root* is ``None``, the object is returned unchanged.
|
|
309
|
+
"""
|
|
310
|
+
if root is None:
|
|
311
|
+
return obj
|
|
312
|
+
root = root.resolve()
|
|
313
|
+
if isinstance(obj, dict):
|
|
314
|
+
return {k: _relativize_paths(v, root) for k, v in obj.items()}
|
|
315
|
+
if isinstance(obj, list):
|
|
316
|
+
return [_relativize_paths(v, root) for v in obj]
|
|
317
|
+
if isinstance(obj, str) and obj.startswith("/"):
|
|
318
|
+
p = Path(obj)
|
|
319
|
+
try:
|
|
320
|
+
return str(p.relative_to(root))
|
|
321
|
+
except ValueError:
|
|
322
|
+
return obj
|
|
323
|
+
return obj
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def run_tasks(
|
|
327
|
+
config: "PipelineConfig",
|
|
328
|
+
tasks: str | Sequence[str] | None = None,
|
|
329
|
+
data_dir: Path | None = None,
|
|
330
|
+
cache_dir: Path | None = None,
|
|
331
|
+
) -> "list[WorkflowResult[Any, Any]]":
|
|
332
|
+
"""Run tasks from a pipeline configuration.
|
|
333
|
+
|
|
334
|
+
Parameters
|
|
335
|
+
----------
|
|
336
|
+
config : PipelineConfig
|
|
337
|
+
Pipeline configuration containing datasets, sources, extractors,
|
|
338
|
+
workflows, and tasks.
|
|
339
|
+
tasks : str | list[str] | None
|
|
340
|
+
Which tasks to run:
|
|
341
|
+
|
|
342
|
+
- ``None`` (default) — run all enabled tasks
|
|
343
|
+
- ``str`` — run a single task by name
|
|
344
|
+
- ``list[str]`` — run specific tasks by name, in the given order
|
|
345
|
+
data_dir : Path | None
|
|
346
|
+
Root directory for resolving relative paths in configs.
|
|
347
|
+
cache_dir : Path | None
|
|
348
|
+
Directory for disk-backed computation cache.
|
|
349
|
+
|
|
350
|
+
Returns
|
|
351
|
+
-------
|
|
352
|
+
list[WorkflowResult]
|
|
353
|
+
One result per task executed, in execution order.
|
|
354
|
+
|
|
355
|
+
Raises
|
|
356
|
+
------
|
|
357
|
+
ValueError
|
|
358
|
+
If no tasks are defined, all are disabled, or a named task is
|
|
359
|
+
not found.
|
|
360
|
+
"""
|
|
361
|
+
if not config.tasks:
|
|
362
|
+
raise ValueError("No tasks defined in pipeline config")
|
|
363
|
+
|
|
364
|
+
if tasks is None:
|
|
365
|
+
# Run all enabled tasks
|
|
366
|
+
to_run = [t for t in config.tasks if t.enabled]
|
|
367
|
+
skipped = len(config.tasks) - len(to_run)
|
|
368
|
+
if skipped:
|
|
369
|
+
logger.info("Skipping %d disabled task(s)", skipped)
|
|
370
|
+
if not to_run:
|
|
371
|
+
raise ValueError("All tasks are disabled — nothing to run")
|
|
372
|
+
elif isinstance(tasks, str):
|
|
373
|
+
to_run = [_resolve_by_name(config.tasks, tasks, "task")]
|
|
374
|
+
else:
|
|
375
|
+
to_run = [_resolve_by_name(config.tasks, name, "task") for name in tasks]
|
|
376
|
+
|
|
377
|
+
logger.info("Running %d task(s)", len(to_run))
|
|
378
|
+
results: list[WorkflowResult[Any, Any]] = []
|
|
379
|
+
for task in to_run:
|
|
380
|
+
logger.info("--- Task: %s (workflow: %s) ---", task.name, task.workflow)
|
|
381
|
+
results.append(_run_single_task(task, config, data_dir=data_dir, cache_dir=cache_dir))
|
|
382
|
+
return results
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
@overload
|
|
386
|
+
def run_task(
|
|
387
|
+
task: "DataAnalysisTaskConfig",
|
|
388
|
+
config: "PipelineConfig",
|
|
389
|
+
data_dir: Path | None = None,
|
|
390
|
+
cache_dir: Path | None = None,
|
|
391
|
+
) -> "WorkflowResult[DataAnalysisMetadata, DataAnalysisOutputs]": ...
|
|
392
|
+
@overload
|
|
393
|
+
def run_task(
|
|
394
|
+
task: "DataCleaningTaskConfig",
|
|
395
|
+
config: "PipelineConfig",
|
|
396
|
+
data_dir: Path | None = None,
|
|
397
|
+
cache_dir: Path | None = None,
|
|
398
|
+
) -> "WorkflowResult[DataCleaningMetadata, DataCleaningOutputs]": ...
|
|
399
|
+
@overload
|
|
400
|
+
def run_task(
|
|
401
|
+
task: "DriftMonitoringTaskConfig",
|
|
402
|
+
config: "PipelineConfig",
|
|
403
|
+
data_dir: Path | None = None,
|
|
404
|
+
cache_dir: Path | None = None,
|
|
405
|
+
) -> "WorkflowResult[DriftMonitoringMetadata, DriftMonitoringOutputs]": ...
|
|
406
|
+
@overload
|
|
407
|
+
def run_task(
|
|
408
|
+
task: "OODDetectionTaskConfig",
|
|
409
|
+
config: "PipelineConfig",
|
|
410
|
+
data_dir: Path | None = None,
|
|
411
|
+
cache_dir: Path | None = None,
|
|
412
|
+
) -> "WorkflowResult[OODDetectionMetadata, OODDetectionOutputs]": ...
|
|
413
|
+
@overload
|
|
414
|
+
def run_task(
|
|
415
|
+
task: "DataPrioritizationTaskConfig",
|
|
416
|
+
config: "PipelineConfig",
|
|
417
|
+
data_dir: Path | None = None,
|
|
418
|
+
cache_dir: Path | None = None,
|
|
419
|
+
) -> "WorkflowResult[DataPrioritizationMetadata, DataPrioritizationOutputs]": ...
|
|
420
|
+
@overload
|
|
421
|
+
def run_task(
|
|
422
|
+
task: "TaskConfig", config: "PipelineConfig", data_dir: Path | None = None, cache_dir: Path | None = None
|
|
423
|
+
) -> "WorkflowResult[Any, Any]": ...
|
|
424
|
+
def run_task(
|
|
425
|
+
task: "TaskConfig", config: "PipelineConfig", data_dir: Path | None = None, cache_dir: Path | None = None
|
|
426
|
+
) -> "WorkflowResult[Any, Any]":
|
|
427
|
+
"""Run a single task, returning a narrowly typed result based on the task type.
|
|
428
|
+
|
|
429
|
+
Unlike :func:`run_tasks`, this function accepts the task config object
|
|
430
|
+
directly rather than looking it up by name, which allows type checkers to
|
|
431
|
+
narrow the return type to the appropriate workflow result type.
|
|
432
|
+
|
|
433
|
+
Parameters
|
|
434
|
+
----------
|
|
435
|
+
task : TaskConfig
|
|
436
|
+
The task configuration to execute.
|
|
437
|
+
config : PipelineConfig
|
|
438
|
+
Pipeline configuration supplying datasets, sources, extractors, and
|
|
439
|
+
workflow definitions. The task does **not** need to appear in
|
|
440
|
+
``config.tasks``.
|
|
441
|
+
data_dir : Path | None
|
|
442
|
+
Root directory for resolving relative paths in configs.
|
|
443
|
+
cache_dir : Path | None
|
|
444
|
+
Directory for disk-backed computation cache.
|
|
445
|
+
|
|
446
|
+
Returns
|
|
447
|
+
-------
|
|
448
|
+
WorkflowResult
|
|
449
|
+
A result typed to the specific workflow — e.g.
|
|
450
|
+
``WorkflowResult[OODDetectionMetadata, OODDetectionOutputs]`` when
|
|
451
|
+
*task* is an :class:`~dataeval_flow.config.OODDetectionTaskConfig`.
|
|
452
|
+
"""
|
|
453
|
+
logger.info("--- Task: %s (workflow: %s) ---", task.name, task.workflow)
|
|
454
|
+
return _run_single_task(task, config, data_dir=data_dir, cache_dir=cache_dir)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Concrete workflow implementations."""
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Data analysis workflow - public re-exports."""
|
|
2
|
+
|
|
3
|
+
from dataeval_flow.workflows.analysis.outputs import (
|
|
4
|
+
BiasResult,
|
|
5
|
+
CrossSplitLabelHealth,
|
|
6
|
+
CrossSplitRedundancy,
|
|
7
|
+
DataAnalysisMetadata,
|
|
8
|
+
DataAnalysisOutputs,
|
|
9
|
+
DataAnalysisRawOutputs,
|
|
10
|
+
DataAnalysisReport,
|
|
11
|
+
DataAnalysisResult,
|
|
12
|
+
DistributionShiftResult,
|
|
13
|
+
ImageQualityResult,
|
|
14
|
+
LabelHealthResult,
|
|
15
|
+
RedundancyResult,
|
|
16
|
+
is_analysis_result,
|
|
17
|
+
)
|
|
18
|
+
from dataeval_flow.workflows.analysis.params import DataAnalysisHealthThresholds, DataAnalysisParameters
|
|
19
|
+
from dataeval_flow.workflows.analysis.workflow import DataAnalysisWorkflow
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"BiasResult",
|
|
23
|
+
"CrossSplitLabelHealth",
|
|
24
|
+
"CrossSplitRedundancy",
|
|
25
|
+
"DataAnalysisHealthThresholds",
|
|
26
|
+
"DataAnalysisMetadata",
|
|
27
|
+
"DataAnalysisOutputs",
|
|
28
|
+
"DataAnalysisParameters",
|
|
29
|
+
"DataAnalysisRawOutputs",
|
|
30
|
+
"DataAnalysisReport",
|
|
31
|
+
"DataAnalysisResult",
|
|
32
|
+
"DataAnalysisWorkflow",
|
|
33
|
+
"DistributionShiftResult",
|
|
34
|
+
"ImageQualityResult",
|
|
35
|
+
"LabelHealthResult",
|
|
36
|
+
"RedundancyResult",
|
|
37
|
+
"is_analysis_result",
|
|
38
|
+
]
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Data analysis workflow outputs."""
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Literal, TypeAlias
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field
|
|
6
|
+
from typing_extensions import TypeIs
|
|
7
|
+
|
|
8
|
+
from dataeval_flow.config.schemas import ResultMetadata
|
|
9
|
+
from dataeval_flow.workflow.base import Reportable, WorkflowOutputsBase, WorkflowReportBase
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from dataeval_flow.workflow import WorkflowResult
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"BiasResult",
|
|
16
|
+
"CrossSplitLabelHealth",
|
|
17
|
+
"CrossSplitRedundancy",
|
|
18
|
+
"CrossSplitResult",
|
|
19
|
+
"DataAnalysisMetadata",
|
|
20
|
+
"DataAnalysisOutputs",
|
|
21
|
+
"DataAnalysisRawOutputs",
|
|
22
|
+
"DataAnalysisReport",
|
|
23
|
+
"DataAnalysisResult",
|
|
24
|
+
"DistributionShiftResult",
|
|
25
|
+
"ImageQualityResult",
|
|
26
|
+
"LabelHealthResult",
|
|
27
|
+
"RedundancyResult",
|
|
28
|
+
"SplitResult",
|
|
29
|
+
"is_analysis_result",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
# Per-split assessment sub-models
|
|
35
|
+
# ---------------------------------------------------------------------------
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ImageQualityResult(BaseModel):
|
|
39
|
+
"""Image-level anomaly detection results."""
|
|
40
|
+
|
|
41
|
+
outlier_count: int = Field(description="Number of items flagged as outliers")
|
|
42
|
+
outlier_rate: float = Field(description="Fraction of items flagged as outliers")
|
|
43
|
+
outlier_summary: dict[str, int] = Field(description="Count of outlier items per metric name")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class RedundancyResult(BaseModel):
|
|
47
|
+
"""Duplicate and near-duplicate detection results."""
|
|
48
|
+
|
|
49
|
+
exact_duplicate_groups: int = Field(description="Number of exact duplicate groups found")
|
|
50
|
+
near_duplicate_groups: int = Field(description="Number of near duplicate groups found")
|
|
51
|
+
exact_duplicates_count: int = Field(description="Total items across all exact duplicate groups")
|
|
52
|
+
near_duplicates_count: int = Field(description="Total items across all near duplicate groups")
|
|
53
|
+
exact_groups: list[list[int]] = Field(default_factory=list, description="Indices per exact duplicate group")
|
|
54
|
+
near_groups: list[list[int]] = Field(default_factory=list, description="Indices per near duplicate group")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class LabelHealthResult(BaseModel):
|
|
58
|
+
"""Label completeness and distribution results."""
|
|
59
|
+
|
|
60
|
+
num_classes: int = Field(description="Number of unique class labels")
|
|
61
|
+
class_distribution: dict[str, int] = Field(description="Mapping of class name to label count")
|
|
62
|
+
empty_images: list[int] = Field(description="Indices of images with no labels")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class BiasResult(BaseModel):
|
|
66
|
+
"""Metadata bias analysis results."""
|
|
67
|
+
|
|
68
|
+
metadata_factors: list[str] = Field(description="Names of metadata factors present")
|
|
69
|
+
metadata_summary: dict[str, dict[str, Any]] = Field(description="Per-factor summary statistics")
|
|
70
|
+
balance_summary: dict[str, Any] | None = Field(default=None, description="Balance (MI) analysis results")
|
|
71
|
+
diversity_summary: dict[str, Any] | None = Field(default=None, description="Diversity analysis results")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# ---------------------------------------------------------------------------
|
|
75
|
+
# Per-split result (composed from assessment sub-models)
|
|
76
|
+
# ---------------------------------------------------------------------------
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class SplitResult(BaseModel):
|
|
80
|
+
"""Per-split summary statistics grouped by assessment area."""
|
|
81
|
+
|
|
82
|
+
num_samples: int = Field(description="Total number of images in the split")
|
|
83
|
+
image_quality: ImageQualityResult = Field(description="Image-level anomaly detection")
|
|
84
|
+
redundancy: RedundancyResult = Field(description="Duplicate detection")
|
|
85
|
+
label_health: LabelHealthResult = Field(description="Label completeness and distribution")
|
|
86
|
+
bias: BiasResult = Field(description="Metadata bias analysis")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
# Cross-split assessment sub-models
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class CrossSplitRedundancy(BaseModel):
|
|
95
|
+
"""Cross-split duplicate leakage detection."""
|
|
96
|
+
|
|
97
|
+
duplicate_leakage: dict[str, Any] = Field(
|
|
98
|
+
default_factory=lambda: {"exact_count": 0, "near_count": 0, "exact_groups": [], "near_groups": []},
|
|
99
|
+
description="Cross-split duplicate detection results",
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class CrossSplitLabelHealth(BaseModel):
|
|
104
|
+
"""Cross-split label comparison results."""
|
|
105
|
+
|
|
106
|
+
label_overlap: dict[str, Any] = Field(
|
|
107
|
+
description="Class-level comparison including shared classes and proportion differences"
|
|
108
|
+
)
|
|
109
|
+
label_parity: dict[str, Any] | None = Field(
|
|
110
|
+
default=None,
|
|
111
|
+
description="Chi-squared label distribution parity test between splits",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class DistributionShiftResult(BaseModel):
|
|
116
|
+
"""Cross-split embedding divergence results."""
|
|
117
|
+
|
|
118
|
+
divergence: float | None = Field(
|
|
119
|
+
default=None,
|
|
120
|
+
description="Embedding-space divergence between splits, or None if no extractor was provided",
|
|
121
|
+
)
|
|
122
|
+
divergence_method: str | None = Field(
|
|
123
|
+
default=None,
|
|
124
|
+
description='Method used for divergence computation ("mst" or "fnn")',
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# ---------------------------------------------------------------------------
|
|
129
|
+
# Cross-split result (composed from assessment sub-models)
|
|
130
|
+
# ---------------------------------------------------------------------------
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class CrossSplitResult(BaseModel):
|
|
134
|
+
"""Comparison results between two dataset splits."""
|
|
135
|
+
|
|
136
|
+
redundancy: CrossSplitRedundancy = Field(description="Cross-split duplicate leakage")
|
|
137
|
+
label_health: CrossSplitLabelHealth = Field(description="Label distribution comparison")
|
|
138
|
+
distribution_shift: DistributionShiftResult = Field(description="Embedding divergence")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# ---------------------------------------------------------------------------
|
|
142
|
+
# Workflow output models
|
|
143
|
+
# ---------------------------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class DataAnalysisRawOutputs(WorkflowOutputsBase):
|
|
147
|
+
"""Machine-readable results from data analysis workflow.
|
|
148
|
+
|
|
149
|
+
``dataset_size`` is the total number of items across all splits.
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
splits: dict[str, SplitResult] = Field(
|
|
153
|
+
default_factory=dict,
|
|
154
|
+
description="Per-split quality summaries keyed by split name",
|
|
155
|
+
)
|
|
156
|
+
cross_split: dict[str, CrossSplitResult] = Field(
|
|
157
|
+
default_factory=dict,
|
|
158
|
+
description='Pairwise cross-split comparisons keyed by "splitA_vs_splitB"',
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class DataAnalysisReport(WorkflowReportBase):
|
|
163
|
+
"""Human-readable report for data analysis workflow."""
|
|
164
|
+
|
|
165
|
+
findings: list[Reportable] = Field(default_factory=list)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class DataAnalysisOutputs(BaseModel):
|
|
169
|
+
"""Complete data analysis workflow output."""
|
|
170
|
+
|
|
171
|
+
raw: DataAnalysisRawOutputs
|
|
172
|
+
report: DataAnalysisReport
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class DataAnalysisMetadata(ResultMetadata):
|
|
176
|
+
"""Metadata for the data-analysis workflow."""
|
|
177
|
+
|
|
178
|
+
mode: Literal["advisory", "preparatory"] = "advisory"
|
|
179
|
+
split_names: list[str] = Field(default_factory=list)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# ---------------------------------------------------------------------------
|
|
183
|
+
# Type alias and TypeIs guard for type narrowing
|
|
184
|
+
# ---------------------------------------------------------------------------
|
|
185
|
+
|
|
186
|
+
#: Fully typed result alias for the data-analysis workflow.
|
|
187
|
+
DataAnalysisResult: TypeAlias = "WorkflowResult[DataAnalysisMetadata, DataAnalysisOutputs]"
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def is_analysis_result(
|
|
191
|
+
result: "WorkflowResult[Any, Any]",
|
|
192
|
+
) -> TypeIs["WorkflowResult[DataAnalysisMetadata, DataAnalysisOutputs]"]:
|
|
193
|
+
"""Narrow a generic ``WorkflowResult`` to a data-analysis result.
|
|
194
|
+
|
|
195
|
+
Useful in the CLI loop or any code that receives a generic result::
|
|
196
|
+
|
|
197
|
+
[result] = run_tasks(config, "my_task")
|
|
198
|
+
if is_analysis_result(result):
|
|
199
|
+
result.metadata.split_names # ✓ typed
|
|
200
|
+
result.data.raw.splits # ✓ typed
|
|
201
|
+
"""
|
|
202
|
+
return isinstance(result.metadata, DataAnalysisMetadata)
|