dataeval-flow 0.1.2__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/.gitignore +6 -0
- dataeval_flow-0.2.1/PKG-INFO +610 -0
- dataeval_flow-0.2.1/README.md +552 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/pyproject.toml +81 -40
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/__init__.py +15 -1
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/__main__.py +40 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/__init__.py +1 -1
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/_discover.py +21 -20
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/_registry.py +13 -7
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/_state.py +9 -6
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_panes/_widgets.py +1 -1
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_screens/_params.py +1 -1
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_screens/_section.py +8 -1
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_viewmodel/_builder_vm.py +5 -4
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_viewmodel/_rendering.py +7 -7
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_viewmodel/_section_vm.py +4 -2
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/app.py +3 -3
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/cli.py +6 -3
- dataeval_flow-0.2.1/src/dataeval_flow/_encoding_cli.py +123 -0
- dataeval_flow-0.2.1/src/dataeval_flow/_logging.py +324 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_version.py +2 -2
- dataeval_flow-0.2.1/src/dataeval_flow/binning.py +545 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/cache.py +219 -159
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/__init__.py +9 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/_loader.py +13 -2
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/_merge.py +10 -0
- dataeval_flow-0.2.1/src/dataeval_flow/config/_models.py +182 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/schemas/__init__.py +18 -2
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/schemas/_dataset.py +12 -7
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/schemas/_extractor.py +18 -1
- dataeval_flow-0.2.1/src/dataeval_flow/config/schemas/_metadata.py +156 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/schemas/_task.py +5 -0
- dataeval_flow-0.1.2/src/dataeval_flow/config/schemas/_selection.py → dataeval_flow-0.2.1/src/dataeval_flow/config/schemas/_view.py +73 -11
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/schemas/_workflow.py +22 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/dataset.py +181 -132
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/embeddings.py +40 -1
- dataeval_flow-0.2.1/src/dataeval_flow/metadata.py +32 -0
- dataeval_flow-0.2.1/src/dataeval_flow/policy.py +417 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/runner.py +55 -6
- dataeval_flow-0.2.1/src/dataeval_flow/view.py +54 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflow/__init__.py +29 -3
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflow/_text_report.py +267 -1
- dataeval_flow-0.2.1/src/dataeval_flow/workflow/base.py +109 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflow/orchestrator.py +149 -30
- dataeval_flow-0.2.1/src/dataeval_flow/workflows/_common.py +131 -0
- dataeval_flow-0.2.1/src/dataeval_flow/workflows/_ontology.py +118 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/analysis/params.py +2 -2
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/analysis/workflow.py +146 -178
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/cleaning/_internal.py +1 -1
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/cleaning/outputs.py +5 -1
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/cleaning/params.py +2 -2
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/cleaning/report.py +3 -3
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/cleaning/workflow.py +22 -11
- dataeval_flow-0.2.1/src/dataeval_flow/workflows/coverage/__init__.py +36 -0
- dataeval_flow-0.2.1/src/dataeval_flow/workflows/coverage/ontology.py +145 -0
- dataeval_flow-0.2.1/src/dataeval_flow/workflows/coverage/outputs.py +356 -0
- dataeval_flow-0.2.1/src/dataeval_flow/workflows/coverage/params.py +231 -0
- dataeval_flow-0.2.1/src/dataeval_flow/workflows/coverage/report.py +595 -0
- dataeval_flow-0.2.1/src/dataeval_flow/workflows/coverage/workflow.py +754 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/drift/workflow.py +12 -10
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/ood/params.py +2 -2
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/ood/workflow.py +85 -29
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/parameter_sweep/workflow.py +3 -3
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/prioritization/params.py +2 -2
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/prioritization/workflow.py +16 -9
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/splitting/report.py +29 -15
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/splitting/workflow.py +11 -29
- dataeval_flow-0.1.2/PKG-INFO +0 -423
- dataeval_flow-0.1.2/README.md +0 -358
- dataeval_flow-0.1.2/src/dataeval_flow/_logging.py +0 -158
- dataeval_flow-0.1.2/src/dataeval_flow/config/_models.py +0 -115
- dataeval_flow-0.1.2/src/dataeval_flow/config/schemas/_metadata.py +0 -28
- dataeval_flow-0.1.2/src/dataeval_flow/metadata.py +0 -48
- dataeval_flow-0.1.2/src/dataeval_flow/selection.py +0 -50
- dataeval_flow-0.1.2/src/dataeval_flow/workflow/base.py +0 -69
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/LICENSE +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/_coerce.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/_execution.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/_introspect.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/_item.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_model/_undo.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_panes/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_panes/_config_pane.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_panes/_result_pane.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_panes/_task_pane.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_screens/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_screens/_base.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_screens/_detail.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_screens/_model.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_screens/_pathpicker.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_screens/_settings.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_viewmodel/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_viewmodel/_model_vm.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/_app/_viewmodel/_result_vm.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/_paths.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/config/schemas/_preprocessor.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/preprocessing.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/preprocessors.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/py.typed +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/analysis/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/analysis/outputs.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/cleaning/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/drift/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/drift/outputs.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/drift/params.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/drift/report.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/ood/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/ood/outputs.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/ood/report.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/parameter_sweep/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/parameter_sweep/outputs.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/parameter_sweep/params.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/prioritization/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/prioritization/outputs.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/prioritization/report.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/splitting/__init__.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/splitting/outputs.py +0 -0
- {dataeval_flow-0.1.2 → dataeval_flow-0.2.1}/src/dataeval_flow/workflows/splitting/params.py +0 -0
|
@@ -37,6 +37,12 @@ output/
|
|
|
37
37
|
# Jupyter execution cache (stored in docs-artifacts/<branch> orphan branch instead)
|
|
38
38
|
.jupyter_cache/
|
|
39
39
|
|
|
40
|
+
# IPython runtime files and configuration
|
|
41
|
+
docs/source/.ipython/*
|
|
42
|
+
!docs/source/.ipython/profile_default/
|
|
43
|
+
docs/source/.ipython/profile_default/*
|
|
44
|
+
!docs/source/.ipython/profile_default/ipython_kernel_config.py
|
|
45
|
+
|
|
40
46
|
# Nox
|
|
41
47
|
.nox/
|
|
42
48
|
.cuda-version
|
|
@@ -0,0 +1,610 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dataeval-flow
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: DataEval Workflows container for data evaluation
|
|
5
|
+
Project-URL: Homepage, https://github.com/aria-ml/dataeval-flow
|
|
6
|
+
Project-URL: Documentation, https://dataeval-flow.readthedocs.io/
|
|
7
|
+
Project-URL: Repository, https://github.com/aria-ml/dataeval-flow
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: <3.15,>=3.10
|
|
21
|
+
Requires-Dist: click>=8.0
|
|
22
|
+
Requires-Dist: dataeval==1.1.0-rc6
|
|
23
|
+
Requires-Dist: datamaite[ic,od]==0.4.1
|
|
24
|
+
Requires-Dist: numpy>=1.24.2
|
|
25
|
+
Requires-Dist: pydantic>=2.0
|
|
26
|
+
Requires-Dist: pyyaml>=6.0
|
|
27
|
+
Provides-Extra: app
|
|
28
|
+
Requires-Dist: textual>=3.0; extra == 'app'
|
|
29
|
+
Provides-Extra: cpu
|
|
30
|
+
Requires-Dist: torch>=2.2.0; extra == 'cpu'
|
|
31
|
+
Requires-Dist: torchvision>=0.17.0; extra == 'cpu'
|
|
32
|
+
Provides-Extra: cu126
|
|
33
|
+
Requires-Dist: torch>=2.2.0; extra == 'cu126'
|
|
34
|
+
Requires-Dist: torchvision>=0.17.0; extra == 'cu126'
|
|
35
|
+
Provides-Extra: cu130
|
|
36
|
+
Requires-Dist: torch>=2.2.0; extra == 'cu130'
|
|
37
|
+
Requires-Dist: torchvision>=0.17.0; extra == 'cu130'
|
|
38
|
+
Provides-Extra: onnx
|
|
39
|
+
Requires-Dist: onnx>=1.15; extra == 'onnx'
|
|
40
|
+
Requires-Dist: onnxruntime<1.24,>=1.20; (python_version == '3.10') and extra == 'onnx'
|
|
41
|
+
Requires-Dist: onnxruntime>=1.20; (python_version >= '3.11') and extra == 'onnx'
|
|
42
|
+
Provides-Extra: onnx-cu126
|
|
43
|
+
Requires-Dist: onnx>=1.15; extra == 'onnx-cu126'
|
|
44
|
+
Requires-Dist: onnxruntime-gpu<1.24,>=1.20; (python_version == '3.10' and extra != 'onnx-cu130') and extra == 'onnx-cu126'
|
|
45
|
+
Requires-Dist: onnxruntime-gpu<1.27,>=1.20; (python_version >= '3.11' and extra != 'onnx-cu130') and extra == 'onnx-cu126'
|
|
46
|
+
Requires-Dist: onnxruntime-gpu<1.27,>=1.24; (python_version >= '3.14' and extra != 'onnx-cu130') and extra == 'onnx-cu126'
|
|
47
|
+
Provides-Extra: onnx-cu130
|
|
48
|
+
Requires-Dist: onnx>=1.15; extra == 'onnx-cu130'
|
|
49
|
+
Requires-Dist: onnxruntime-gpu>=1.27; (python_version >= '3.11' and extra != 'onnx-cu126') and extra == 'onnx-cu130'
|
|
50
|
+
Requires-Dist: onnxruntime>=1.20; (python_version == '3.10' and extra != 'onnx-cu126') and extra == 'onnx-cu130'
|
|
51
|
+
Provides-Extra: ontology
|
|
52
|
+
Requires-Dist: dataeval[ontology]==1.1.0-rc6; extra == 'ontology'
|
|
53
|
+
Provides-Extra: opencv
|
|
54
|
+
Requires-Dist: opencv-python-headless>=4.8.0; extra == 'opencv'
|
|
55
|
+
Provides-Extra: opencv-fips
|
|
56
|
+
Requires-Dist: opencv-python-headless<4.13.0,>=4.8.0; extra == 'opencv-fips'
|
|
57
|
+
Description-Content-Type: text/markdown
|
|
58
|
+
|
|
59
|
+
<!-- markdownlint-disable MD041 -->
|
|
60
|
+

|
|
61
|
+
|
|
62
|
+
# DataEval Flow
|
|
63
|
+
|
|
64
|
+
DataEval Flow provides workflow orchestration for DataEval evaluators, packaging
|
|
65
|
+
data cleaning, dataset analysis, coverage assessment, drift monitoring, OOD
|
|
66
|
+
detection, splitting, prioritization, and parameter-sweep pipelines behind a
|
|
67
|
+
single declarative configuration format and both headless and interactive CLIs.
|
|
68
|
+
|
|
69
|
+
📖 **Documentation:** <https://dataeval-flow.readthedocs.io/>
|
|
70
|
+
|
|
71
|
+
## What DataEval Flow is for
|
|
72
|
+
|
|
73
|
+
<!-- start needs -->
|
|
74
|
+
|
|
75
|
+
DataEval Flow lets T&E engineers compose and run multi-stage data evaluation
|
|
76
|
+
pipelines without writing Python glue code. Pipelines are described in YAML or
|
|
77
|
+
JSON, executed locally or in a CUDA-enabled container, and produce both
|
|
78
|
+
human-readable reports and machine-readable result envelopes that satisfy JATIC
|
|
79
|
+
interoperability requirements. It builds directly on the
|
|
80
|
+
[DataEval](https://dataeval.readthedocs.io/) library, so the underlying
|
|
81
|
+
evaluators — outlier and duplicate detection, coverage and gap analysis, drift
|
|
82
|
+
and OOD monitoring, dataset splitting, prioritization, and statistical analysis —
|
|
83
|
+
are the same algorithms DataEval exposes, wrapped in a reproducible
|
|
84
|
+
orchestration layer.
|
|
85
|
+
|
|
86
|
+
<!-- end needs -->
|
|
87
|
+
|
|
88
|
+
## Target Audience
|
|
89
|
+
|
|
90
|
+
<!-- start JATIC interop -->
|
|
91
|
+
|
|
92
|
+
DataEval Flow is intended for data scientists, ML engineers, and T&E engineers
|
|
93
|
+
who want to run automated data-evaluation pipelines against their image datasets
|
|
94
|
+
and models. It is part of the JATIC suite of tools: DataEval Flow has native
|
|
95
|
+
interoperability when using MAITE-compliant datasets and models, so its outputs
|
|
96
|
+
compose with other MAITE-conforming JATIC tools.
|
|
97
|
+
|
|
98
|
+
<!-- end JATIC interop -->
|
|
99
|
+
|
|
100
|
+
## Limitations and requirements for use
|
|
101
|
+
|
|
102
|
+
- **Computer-vision image datasets only.** DataEval Flow operates on image
|
|
103
|
+
classification and object-detection datasets; it does not handle NLP or
|
|
104
|
+
tabular data.
|
|
105
|
+
- **MAITE for native interoperability.** Non-MAITE sources are consumed through
|
|
106
|
+
the built-in adapters (HuggingFace, COCO, YOLO, TorchVision, ImageFolder);
|
|
107
|
+
native JATIC interoperability requires MAITE-compliant datasets/models.
|
|
108
|
+
- **Some workflows need metadata.** Bias, parity, and metadata-insight analyses
|
|
109
|
+
require per-sample metadata factors to be present in the dataset.
|
|
110
|
+
- **Some workflows need a model or embeddings.** Embedding-space drift, OOD
|
|
111
|
+
detection, and prioritization require a feature extractor (ONNX/PyTorch) or
|
|
112
|
+
precomputed embeddings.
|
|
113
|
+
- **Drift and OOD need a representative reference.** Detection baselines are only
|
|
114
|
+
as good as the reference dataset they are fit on.
|
|
115
|
+
- **Batch container, not a service.** The container runs a pipeline to
|
|
116
|
+
completion and exits; it is not a long-running web service (no health-check
|
|
117
|
+
endpoint).
|
|
118
|
+
|
|
119
|
+
## System Requirements
|
|
120
|
+
|
|
121
|
+
The guidance below applies to both the container and the Python-library forms.
|
|
122
|
+
|
|
123
|
+
### Tested platforms
|
|
124
|
+
|
|
125
|
+
DataEval Flow is developed and tested on Linux (Ubuntu 22.04 and 24.04, including
|
|
126
|
+
WSL2). The Python package supports **Python 3.10–3.14**; the CI test matrix runs
|
|
127
|
+
all five. The container images are all built on Ubuntu 24.04 (cpu, cu126, cu130).
|
|
128
|
+
macOS and Windows are supported only through Docker or
|
|
129
|
+
WSL2 and are not part of the CI test matrix — if you hit an issue on those hosts,
|
|
130
|
+
the OS/hardware may be the root cause.
|
|
131
|
+
|
|
132
|
+
### Architecture
|
|
133
|
+
|
|
134
|
+
All container images and the dependency stack target **linux/amd64 (x86-64)**.
|
|
135
|
+
arm64 / Apple Silicon is not built or tested; on those hosts run the CPU image
|
|
136
|
+
under emulation or install the library from source. The `dataeval_flow` package
|
|
137
|
+
ships no compiled extensions of its own, so the library form runs anywhere its
|
|
138
|
+
dependencies (PyTorch, NumPy, SciPy) provide x86-64 wheels.
|
|
139
|
+
|
|
140
|
+
### Recommended minimum hardware
|
|
141
|
+
|
|
142
|
+
| Resource | Minimum | Recommended | Notes |
|
|
143
|
+
| -------- | --------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
144
|
+
| CPU | 2 cores | 4+ cores | Dataset loading and statistical analysis are CPU-bound. |
|
|
145
|
+
| Memory | 8 GB | 16+ GB | Datasets and embeddings are held in memory during a run; peak scales with dataset and batch size. **Memory is the primary limit on dataset size.** |
|
|
146
|
+
| Disk | 10 GB | 20+ GB | Several GB for the container image / dependencies, plus dataset and `/cache` storage. |
|
|
147
|
+
| GPU | none (optional) | NVIDIA, ≥ 4 GB VRAM | Optional — used only to accelerate model-based embedding extraction (ONNX / PyTorch). Every workflow runs CPU-only via the `cpu` image or a CPU torch build; a GPU speeds up embedding-heavy workflows (drift / OOD / prioritization) on large datasets. |
|
|
148
|
+
|
|
149
|
+
A GPU is never required. When deploying the container under Kubernetes, request
|
|
150
|
+
at least the minimum CPU/memory above; size memory to your largest dataset.
|
|
151
|
+
|
|
152
|
+
### Internet access
|
|
153
|
+
|
|
154
|
+
- **Installation** needs network access to PyPI and the PyTorch wheel index (or,
|
|
155
|
+
for the container, to the base image and the Harbor registry).
|
|
156
|
+
- **First run** downloads any datasets referenced from the HuggingFace Hub (and,
|
|
157
|
+
in the tutorials, sample datasets such as MNIST / CPPE-5). Model weights
|
|
158
|
+
referenced by URL are likewise fetched on first use.
|
|
159
|
+
- **Offline / air-gapped operation** is supported once the image, datasets, and
|
|
160
|
+
models are staged locally: point the config at on-disk dataset/model paths and
|
|
161
|
+
set `HF_HUB_OFFLINE=1` (and `HF_DATASETS_OFFLINE=1`). With local inputs the
|
|
162
|
+
batch container makes no outbound network calls of its own at run time.
|
|
163
|
+
|
|
164
|
+
## Quick Start
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
# 1. Build CUDA 12.6 container
|
|
168
|
+
docker build -f docker/Dockerfile.cu126 -t dataeval:cu126 .
|
|
169
|
+
|
|
170
|
+
# 2. Show help
|
|
171
|
+
docker run dataeval:cu126
|
|
172
|
+
|
|
173
|
+
# 3. Run with data and output
|
|
174
|
+
docker run --gpus all \
|
|
175
|
+
--mount type=bind,source=/path/to/data,target=/dataeval,readonly \
|
|
176
|
+
--mount type=bind,source=/path/to/output,target=/output \
|
|
177
|
+
dataeval:cu126
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
## Pulling pre-built images
|
|
181
|
+
|
|
182
|
+
Pre-built, cosign-signed images are published to Harbor for every merge to
|
|
183
|
+
`main` and every release tag. Pull one of these instead of building from source
|
|
184
|
+
if you don't need to modify the code.
|
|
185
|
+
|
|
186
|
+
**Rolling channel** — tracks the latest commit on `main`. The tag is overwritten on every merge.
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
docker pull harbor.jatic.net/aria/dataeval:cu126 # cpu / cu126 / cu130
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
**Pinned release channel** — immutable, version-tagged images cut from `v*` git tags. Use these for reproducible workloads.
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
docker pull harbor.jatic.net/aria/dataeval:0.1.0-cu126
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
**Verifying the signature** — every published image is signed with
|
|
199
|
+
[cosign](https://docs.sigstore.dev/cosign/). The public key is committed at
|
|
200
|
+
[docker/cosign.pub](docker/cosign.pub).
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
cosign verify --key docker/cosign.pub harbor.jatic.net/aria/dataeval:cu126
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Then drop the `dataeval:cu126` reference in the Quick Start `docker run`
|
|
207
|
+
commands above with the fully-qualified `harbor.jatic.net/aria/dataeval:cu126`
|
|
208
|
+
(or pinned version) and skip step 1.
|
|
209
|
+
|
|
210
|
+
> **Note on feature branches.** Containers are only built and published from
|
|
211
|
+
> `main` and release tags — no image is produced for MRs or topic branches. If
|
|
212
|
+
> you want to run a feature branch as a container, check it out and follow the
|
|
213
|
+
> Quick Start to build locally; the resulting image will pick up the branch's
|
|
214
|
+
> version via `git describe`.
|
|
215
|
+
|
|
216
|
+
## Requirements
|
|
217
|
+
|
|
218
|
+
| Requirement | Version |
|
|
219
|
+
| ------------- | --------------------- |
|
|
220
|
+
| Docker | >= 20.10 |
|
|
221
|
+
| NVIDIA GPU | Any (for GPU mode) |
|
|
222
|
+
| NVIDIA Driver | >= 520 (for GPU mode) |
|
|
223
|
+
| CUDA | 11.8.0 (for GPU mode) |
|
|
224
|
+
|
|
225
|
+
### Verify GPU Access
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
docker run --rm --gpus all nvidia/cuda:11.8.0-base-ubuntu22.04 nvidia-smi
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
## Volume Mounts
|
|
232
|
+
|
|
233
|
+
| Path | Mode | Purpose |
|
|
234
|
+
| ----------- | ---- | ----------------------------------------------------- |
|
|
235
|
+
| `/dataeval` | ro | Data directory — datasets, models, configs (required) |
|
|
236
|
+
| `/output` | rw | Results (required) |
|
|
237
|
+
| `/cache` | rw | Computation cache (optional) |
|
|
238
|
+
|
|
239
|
+
### File Permissions
|
|
240
|
+
|
|
241
|
+
The container runs as a non-root user (`dataeval`, UID 1000). Mounted
|
|
242
|
+
directories for `/output` and `/cache` must be writable by the container
|
|
243
|
+
process. There are two approaches:
|
|
244
|
+
|
|
245
|
+
#### Option 1: Pass your host UID (recommended)
|
|
246
|
+
|
|
247
|
+
Use `--user` to run the container as your host user, so mounted directories are naturally writable:
|
|
248
|
+
|
|
249
|
+
```bash
|
|
250
|
+
docker run --gpus all \
|
|
251
|
+
--user "$(id -u):$(id -g)" \
|
|
252
|
+
--mount type=bind,source=/path/to/data,target=/dataeval,readonly \
|
|
253
|
+
--mount type=bind,source=/path/to/output,target=/output \
|
|
254
|
+
dataeval:cu126
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
#### Option 2: Open directory permissions
|
|
258
|
+
|
|
259
|
+
Make the output and cache directories world-writable on the host:
|
|
260
|
+
|
|
261
|
+
```bash
|
|
262
|
+
chmod 777 /path/to/output /path/to/cache
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
Then run without `--user`. This is simpler but less secure.
|
|
266
|
+
|
|
267
|
+
### Custom Data Root
|
|
268
|
+
|
|
269
|
+
The data root path can be overridden via the `DATAEVAL_DATA` environment variable:
|
|
270
|
+
|
|
271
|
+
```bash
|
|
272
|
+
docker run --gpus all \
|
|
273
|
+
-e DATAEVAL_DATA=/data \
|
|
274
|
+
--mount type=bind,source=/path/to/data,target=/data,readonly \
|
|
275
|
+
--mount type=bind,source=/path/to/output,target=/output \
|
|
276
|
+
dataeval:cu126
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
## Environment Variables
|
|
280
|
+
|
|
281
|
+
All runtime environment variables are optional; command-line options take
|
|
282
|
+
precedence over them (see [Input Precedence](#input-precedence) below).
|
|
283
|
+
|
|
284
|
+
| Variable | Purpose | Default |
|
|
285
|
+
| ----------------- | ----------------------------------------------- | --------------------------------------------------------- |
|
|
286
|
+
| `DATAEVAL_DATA` | Input data root — datasets, models, and configs | `/dataeval` in the container; current directory otherwise |
|
|
287
|
+
| `DATAEVAL_OUTPUT` | Directory for results and reports | `/output` in the container |
|
|
288
|
+
| `DATAEVAL_CACHE` | Disk-backed computation cache (optional) | `/cache` when that mount is present and writable |
|
|
289
|
+
|
|
290
|
+
No secret mounts or credentials are required — DataEval Flow uses no API keys,
|
|
291
|
+
tokens, or passwords. (`DATAEVAL_FLOW_VERSION` and `DATAEVAL_NOX_UV_EXTRAS_OVERRIDE`
|
|
292
|
+
are build-time only and are not read at run time.)
|
|
293
|
+
|
|
294
|
+
## Input Precedence
|
|
295
|
+
|
|
296
|
+
For any input, the resolution order is:
|
|
297
|
+
|
|
298
|
+
1. **Command-line option** — `--config`, `--data`, `--output`, `--cache`
|
|
299
|
+
2. **Environment variable** — `DATAEVAL_DATA`, `DATAEVAL_OUTPUT`, `DATAEVAL_CACHE`
|
|
300
|
+
3. **Built-in default** — the container mount paths above (or the current
|
|
301
|
+
directory outside the container)
|
|
302
|
+
|
|
303
|
+
Dataset and model paths inside a config file are resolved relative to the data
|
|
304
|
+
root; a relative path not found directly is also looked up under the conventional
|
|
305
|
+
`data/` (datasets) and `models/` (models) subfolders of the data root.
|
|
306
|
+
|
|
307
|
+
## Interface Documentation
|
|
308
|
+
|
|
309
|
+
The container prints its full interface — mounts, environment variables, CLI
|
|
310
|
+
options, precedence, and examples — via its help command, which is also the
|
|
311
|
+
default when the container runs with no pipeline arguments:
|
|
312
|
+
|
|
313
|
+
```bash
|
|
314
|
+
docker run dataeval:cu126 --help
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
The library form exposes the same options via `python -m dataeval_flow --help`.
|
|
318
|
+
The published [container reference](https://dataeval-flow.readthedocs.io/en/latest/reference/containers.html)
|
|
319
|
+
documents every input, default, and configuration dependency.
|
|
320
|
+
|
|
321
|
+
## Configuration
|
|
322
|
+
|
|
323
|
+
Config files (YAML or JSON) can be placed anywhere in your data directory. By
|
|
324
|
+
default, all YAML/JSON files at the root of the data mount are auto-discovered
|
|
325
|
+
and merged.
|
|
326
|
+
|
|
327
|
+
To specify a config path explicitly:
|
|
328
|
+
|
|
329
|
+
```bash
|
|
330
|
+
# Config folder within data directory
|
|
331
|
+
docker run --gpus all \
|
|
332
|
+
--mount type=bind,source=/path/to/data,target=/dataeval,readonly \
|
|
333
|
+
--mount type=bind,source=/path/to/output,target=/output \
|
|
334
|
+
dataeval:cu126 --config config/
|
|
335
|
+
|
|
336
|
+
# Single config file
|
|
337
|
+
docker run --gpus all \
|
|
338
|
+
--mount type=bind,source=/path/to/data,target=/dataeval,readonly \
|
|
339
|
+
--mount type=bind,source=/path/to/output,target=/output \
|
|
340
|
+
dataeval:cu126 --config params.yaml
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
Dataset and model paths in config files are resolved relative to the data root (`/dataeval` by default).
|
|
344
|
+
|
|
345
|
+
## Dataset Formats
|
|
346
|
+
|
|
347
|
+
Currently supported dataset structures:
|
|
348
|
+
|
|
349
|
+
| Format | Structure | Example |
|
|
350
|
+
| --------------- | -------------------------------------------------- | --------------- |
|
|
351
|
+
| **Dataset** | Single split, used directly | `cifar10_test/` |
|
|
352
|
+
| **DatasetDict** | Multiple splits (dict), configured via config YAML | `cifar10_full/` |
|
|
353
|
+
|
|
354
|
+
## CPU Fallback
|
|
355
|
+
|
|
356
|
+
For machines without NVIDIA GPU:
|
|
357
|
+
|
|
358
|
+
```bash
|
|
359
|
+
docker build -f docker/Dockerfile.cpu -t dataeval:cpu .
|
|
360
|
+
docker run dataeval:cpu # Shows help
|
|
361
|
+
docker run \
|
|
362
|
+
--mount type=bind,source=/path/to/data,target=/dataeval,readonly \
|
|
363
|
+
--mount type=bind,source=/path/to/output,target=/output \
|
|
364
|
+
dataeval:cpu
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
## CLI Modes
|
|
368
|
+
|
|
369
|
+
DataEval Flow has three modes:
|
|
370
|
+
|
|
371
|
+
| Command | Purpose |
|
|
372
|
+
| ---------------------- | ---------------------------------------------------------------- |
|
|
373
|
+
| `dataeval-flow [opts]` | Headless execution — for automation and CI/CD pipelines |
|
|
374
|
+
| `dataeval-flow app` | Interactive TUI dashboard — configure, execute, and view results |
|
|
375
|
+
| `dataeval-flow config` | Simple CLI config builder — create/edit configs without the TUI |
|
|
376
|
+
|
|
377
|
+
### Interactive TUI (`app`)
|
|
378
|
+
|
|
379
|
+
**Installation:**
|
|
380
|
+
|
|
381
|
+
```bash
|
|
382
|
+
uv sync --extra app # or: pip install dataeval-flow[app]
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
**Usage:**
|
|
386
|
+
|
|
387
|
+
```bash
|
|
388
|
+
# Launch with a blank config
|
|
389
|
+
python -m dataeval_flow app
|
|
390
|
+
|
|
391
|
+
# Load an existing config for editing
|
|
392
|
+
python -m dataeval_flow app --config /path/to/params.yaml
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
The TUI provides a three-pane dashboard for config editing, task execution, and
|
|
396
|
+
result viewing. It auto-discovers available torchvision transforms, dataeval
|
|
397
|
+
selection classes, and workflow types, generating dynamic parameter forms from
|
|
398
|
+
their schemas.
|
|
399
|
+
|
|
400
|
+
### Simple CLI Config Builder (`config`)
|
|
401
|
+
|
|
402
|
+
For environments without the TUI dependency:
|
|
403
|
+
|
|
404
|
+
```bash
|
|
405
|
+
python -m dataeval_flow config
|
|
406
|
+
python -m dataeval_flow config --config /path/to/params.yaml
|
|
407
|
+
```
|
|
408
|
+
|
|
409
|
+
Configs can be saved as YAML or JSON.
|
|
410
|
+
|
|
411
|
+
## Dependencies
|
|
412
|
+
|
|
413
|
+
- `dataeval` - Core evaluation library
|
|
414
|
+
- `datamaite` - MAITE protocol dataset loaders
|
|
415
|
+
- `maite` - MAITE protocol library
|
|
416
|
+
- `pydantic` - Structural typing and schema validation
|
|
417
|
+
|
|
418
|
+
## Troubleshooting
|
|
419
|
+
|
|
420
|
+
### Build appears stuck at `uv sync`
|
|
421
|
+
|
|
422
|
+
The Docker build may appear frozen during the `uv sync` step:
|
|
423
|
+
|
|
424
|
+
```text
|
|
425
|
+
=> [builder 7/7] RUN uv sync --frozen --no-dev --no-install-project 1139.3s
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
**This is normal.** The step downloads ~2GB of dependencies (PyTorch, scipy, etc.) with no progress indicator.
|
|
429
|
+
|
|
430
|
+
| Network Speed | Expected Build Time |
|
|
431
|
+
| ------------- | ------------------- |
|
|
432
|
+
| 100 Mbps | ~10 minutes |
|
|
433
|
+
| 30 Mbps | ~20 minutes |
|
|
434
|
+
| 10 Mbps | ~45 minutes |
|
|
435
|
+
|
|
436
|
+
**Tip:** First build is slow; subsequent builds use Docker cache and complete in seconds.
|
|
437
|
+
|
|
438
|
+
## Running Without Container
|
|
439
|
+
|
|
440
|
+
The `dataeval_flow` package can be used standalone without Docker.
|
|
441
|
+
|
|
442
|
+
**Installation:**
|
|
443
|
+
|
|
444
|
+
See the [Installation guide](https://dataeval-flow.readthedocs.io/en/latest/home/installation.html)
|
|
445
|
+
for the full matrix. In short:
|
|
446
|
+
|
|
447
|
+
`pip` from PyPI (no source checkout). PyTorch arrives transitively via `dataeval`.
|
|
448
|
+
To choose a variant, install `torch` from that variant's index **first**, then install
|
|
449
|
+
DataEval Flow — it accepts the build already present (omit step 1 and you'll get the
|
|
450
|
+
CUDA-bundled manylinux build of torch from PyPI, which is much larger):
|
|
451
|
+
|
|
452
|
+
```bash
|
|
453
|
+
# 1. Pick your PyTorch build (cpu / cu126 / cu130)
|
|
454
|
+
pip install torch torchvision --index-url https://download.pytorch.org/whl/cu130
|
|
455
|
+
|
|
456
|
+
# 2. Install DataEval Flow
|
|
457
|
+
pip install dataeval-flow
|
|
458
|
+
```
|
|
459
|
+
|
|
460
|
+
> **Use `--index-url`, not `--extra-index-url`, to pick a CUDA build.**
|
|
461
|
+
> `--extra-index-url` *adds* an index instead of replacing PyPI, and pip then takes the
|
|
462
|
+
> highest version across both. The CUDA indexes lag the latest PyTorch release, so PyPI
|
|
463
|
+
> usually wins and you silently get the default CUDA-bundled build — the install
|
|
464
|
+
> succeeds with no warning. `--index-url` replaces the index outright, so it is
|
|
465
|
+
> reliable. (For CPU only, `pip install dataeval-flow --extra-index-url
|
|
466
|
+
> https://download.pytorch.org/whl/cpu` does work, because the CPU index tracks the
|
|
467
|
+
> latest release.)
|
|
468
|
+
|
|
469
|
+
`torchvision` is not installed by default — it is imported lazily and is only needed
|
|
470
|
+
for preprocessing pipelines, the torchvision dataset adapter, and the TUI's transform
|
|
471
|
+
discovery. Install it together with `torch` in step 1 so both come from the same index.
|
|
472
|
+
Feature extras (`onnx`, `onnx-cu126`, `onnx-cu130`, `opencv`, `app`, `ontology`) work normally under
|
|
473
|
+
pip and are independent of the PyTorch variant:
|
|
474
|
+
|
|
475
|
+
```bash
|
|
476
|
+
pip install "dataeval-flow[onnx,opencv,app]"
|
|
477
|
+
```
|
|
478
|
+
|
|
479
|
+
> **The `cpu` / `cu126` / `cu130` extras do not select a PyTorch variant under pip.**
|
|
480
|
+
> All three declare the same requirements (`torch`, `torchvision`); what distinguishes
|
|
481
|
+
> them is `[tool.uv.sources]`, which routes those packages to the right wheel index.
|
|
482
|
+
> That is project metadata applied by uv when resolving **from source** — it is not
|
|
483
|
+
> part of the published wheel. Under pip their only real effect is pulling in
|
|
484
|
+
> `torchvision`. Select the variant with `--index-url` under pip, `--torch-backend`
|
|
485
|
+
> under `uv pip`, and use the extras only for source installs.
|
|
486
|
+
|
|
487
|
+
`uv` from PyPI:
|
|
488
|
+
|
|
489
|
+
```bash
|
|
490
|
+
uv pip install dataeval-flow --torch-backend cpu # or cu126 / cu130 / auto
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
`uv` from source (default toolchain; uses committed `uv.lock`) — extras apply here:
|
|
494
|
+
|
|
495
|
+
```bash
|
|
496
|
+
git clone https://github.com/aria-ml/dataeval-flow.git
|
|
497
|
+
cd dataeval-flow
|
|
498
|
+
uv sync --extra cpu # or cu126 / cu130; add --extra onnx --extra opencv --extra app as needed
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
`poetry` (source checkout; uses committed `poetry.lock`):
|
|
502
|
+
|
|
503
|
+
```bash
|
|
504
|
+
git clone https://github.com/aria-ml/dataeval-flow.git
|
|
505
|
+
cd dataeval-flow
|
|
506
|
+
poetry install
|
|
507
|
+
```
|
|
508
|
+
|
|
509
|
+
`conda` / `mamba` (source checkout; uses committed `environment.yml`):
|
|
510
|
+
|
|
511
|
+
```bash
|
|
512
|
+
git clone https://github.com/aria-ml/dataeval-flow.git
|
|
513
|
+
cd dataeval-flow
|
|
514
|
+
conda env create -f environment.yml
|
|
515
|
+
conda activate dataeval-flow
|
|
516
|
+
pip install -e . # install the package itself; conda manages deps
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
Notes:
|
|
520
|
+
|
|
521
|
+
- PyTorch is installed from PyPI/`download.pytorch.org` in every path
|
|
522
|
+
(it is no longer maintained on conda-forge).
|
|
523
|
+
- GPU variants (`cu126`, `cu130`) are only wired through `uv` and
|
|
524
|
+
`pip` today; the Poetry/conda paths install the CPU build of PyTorch.
|
|
525
|
+
|
|
526
|
+
**CLI Usage:**
|
|
527
|
+
|
|
528
|
+
```bash
|
|
529
|
+
python -m dataeval_flow --config /path/to/config --output /path/to/output
|
|
530
|
+
python -m dataeval_flow --data /path/to/data --output /path/to/output
|
|
531
|
+
```
|
|
532
|
+
|
|
533
|
+
**Python API Usage:**
|
|
534
|
+
|
|
535
|
+
```python
|
|
536
|
+
from pathlib import Path
|
|
537
|
+
from dataeval_flow import load_config, run_tasks
|
|
538
|
+
|
|
539
|
+
config = load_config(Path("/path/to/data/config.yaml"))
|
|
540
|
+
results = run_tasks(config, data_dir=Path("/path/to/data"))
|
|
541
|
+
print(results[0].report())
|
|
542
|
+
```
|
|
543
|
+
|
|
544
|
+
**Development:**
|
|
545
|
+
|
|
546
|
+
```bash
|
|
547
|
+
uvx --with nox-uv nox -s dev # build .venv with the dev toolchain
|
|
548
|
+
source .venv/bin/activate
|
|
549
|
+
nox # run the default sessions
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
`nox -s dev` prompts for the Python version and device variant, records the choice
|
|
553
|
+
in `.cuda-version` so the other sessions match it, and installs the `onnx` and
|
|
554
|
+
`app` extras alongside. See
|
|
555
|
+
[Development Setup](./CONTRIBUTING.md#development-setup) for the arguments it takes.
|
|
556
|
+
|
|
557
|
+
## Versioning
|
|
558
|
+
|
|
559
|
+
The package version is **derived from git tags** — there is no hardcoded version
|
|
560
|
+
anywhere in the source tree. `hatch-vcs` reads `git describe --tags` at
|
|
561
|
+
build/install time and writes the resolved version to a generated
|
|
562
|
+
`src/dataeval_flow/_version.py` (gitignored), which `dataeval_flow.__init__`
|
|
563
|
+
imports at runtime.
|
|
564
|
+
|
|
565
|
+
**Release flow for container images:**
|
|
566
|
+
|
|
567
|
+
1. Push a semver tag (e.g. `v0.2.0`) — this is the single source of truth for the release version.
|
|
568
|
+
2. The `push:docker` CI job runs `git describe --tags --always --dirty | sed
|
|
569
|
+
's/^v//'` to resolve `${VERSION}`, then passes `--build-arg
|
|
570
|
+
DATAEVAL_FLOW_VERSION="${VERSION}"` to `docker buildx build` for both the
|
|
571
|
+
`test` and `prod` stages.
|
|
572
|
+
3. The `prod` stage in [docker/Dockerfile.j2](docker/Dockerfile.j2) redeclares the ARG and:
|
|
573
|
+
- Bakes the resolved version into `/app/src/dataeval_flow/_version.py` so
|
|
574
|
+
`dataeval_flow.__version__` matches the wheel version at runtime.
|
|
575
|
+
- Stamps the OCI `org.opencontainers.image.version` label with the same value.
|
|
576
|
+
4. The image is pushed to Harbor and cosign-signed.
|
|
577
|
+
|
|
578
|
+
The `ARG DATAEVAL_FLOW_VERSION="…"` default rendered into each committed
|
|
579
|
+
`docker/Dockerfile.<variant>` by `docker/generate.py` is only used for **local**
|
|
580
|
+
`docker build` invocations that don't pass `--build-arg`. Release builds always
|
|
581
|
+
override it, so the committed default is allowed to drift from the latest tag and
|
|
582
|
+
does not need to be regenerated at release time.
|
|
583
|
+
|
|
584
|
+
## License
|
|
585
|
+
|
|
586
|
+
MIT — see [LICENSE](LICENSE).
|
|
587
|
+
|
|
588
|
+
## Contributing
|
|
589
|
+
|
|
590
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for bug reports and contribution
|
|
591
|
+
guidelines, and [BRANCHING.md](BRANCHING.md) for the branching and release
|
|
592
|
+
strategy.
|
|
593
|
+
|
|
594
|
+
## Contact
|
|
595
|
+
|
|
596
|
+
For questions or feedback, reach out to <dataeval-flow@ariacoustics.com>.
|
|
597
|
+
|
|
598
|
+
## Acknowledgement
|
|
599
|
+
|
|
600
|
+
### CDAO Funding Acknowledgement
|
|
601
|
+
|
|
602
|
+
<!-- start acknowledgement -->
|
|
603
|
+
|
|
604
|
+
This material is based upon work supported by the Chief Digital and Artificial
|
|
605
|
+
Intelligence Office under Contract No. W519TC-23-9-2033. The views and
|
|
606
|
+
conclusions contained herein are those of the author(s) and should not be
|
|
607
|
+
interpreted as necessarily representing the official policies or endorsements,
|
|
608
|
+
either expressed or implied, of the U.S. Government.
|
|
609
|
+
|
|
610
|
+
<!-- end acknowledgement -->
|