dataeval 1.1.5__tar.gz → 1.2.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/.gitignore +4 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/PKG-INFO +27 -44
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/README.md +23 -37
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/pyproject.toml +42 -58
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/__init__.py +13 -5
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_embeddings.py +45 -28
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_experimental.py +37 -3
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_helpers.py +464 -126
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_log.py +8 -1
- dataeval-1.2.0rc1/src/dataeval/_metadata/_aggregate.py +670 -0
- dataeval-1.2.0rc1/src/dataeval/_metadata/_collisions.py +108 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_columns.py +67 -11
- dataeval-1.2.0rc1/src/dataeval/_metadata/_corrections.py +289 -0
- dataeval-1.2.0rc1/src/dataeval/_metadata/_encoding.py +685 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_filters.py +10 -1
- dataeval-1.2.0rc1/src/dataeval/_metadata/_input.py +195 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_keyed.py +68 -12
- dataeval-1.2.0rc1/src/dataeval/_metadata/_known.py +119 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_links.py +62 -6
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_loading.py +25 -23
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_metadata.py +2937 -466
- dataeval-1.2.0rc1/src/dataeval/_metadata/_reductions.py +721 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_serialize.py +188 -1
- dataeval-1.2.0rc1/src/dataeval/_metadata/_source_groups.py +83 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_store.py +29 -14
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/__init__.py +6 -4
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_accumulator.py +27 -16
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_base.py +23 -8
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_classification.py +32 -15
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_data.py +13 -2
- dataeval-1.2.0rc1/src/dataeval/_metadata/_structurers/_dataset.py +222 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_detection.py +10 -15
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_factors.py +38 -30
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_frames.py +9 -2
- dataeval-1.2.0rc1/src/dataeval/_metadata/_structurers/_instances.py +36 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_layout.py +1 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_reporting.py +2 -2
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_reserved.py +81 -14
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_select.py +28 -32
- dataeval-1.2.0rc1/src/dataeval/_metadata/_structurers/_source_index.py +316 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_tracking.py +144 -41
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_ontology.py +4 -6
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_version.py +2 -2
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/bias/_balance.py +133 -40
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/bias/_diversity.py +88 -19
- dataeval-1.2.0rc1/src/dataeval/bias/_parity.py +610 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/__init__.py +104 -3
- dataeval-1.2.0rc1/src/dataeval/core/_alignment.py +350 -0
- dataeval-1.2.0rc1/src/dataeval/core/_annotation_match.py +400 -0
- dataeval-1.2.0rc1/src/dataeval/core/_ap_bounds.py +197 -0
- dataeval-1.2.0rc1/src/dataeval/core/_association.py +274 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_ber.py +99 -36
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_bin.py +1 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/_base.py +8 -10
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/_cache.py +10 -45
- dataeval-1.2.0rc1/src/dataeval/core/_calculators/_descriptivestats.py +43 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/_dimensionstats.py +63 -1
- dataeval-1.2.0rc1/src/dataeval/core/_calculators/_egostats.py +61 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/_hashstats.py +16 -14
- dataeval-1.2.0rc1/src/dataeval/core/_calculators/_persistencestats.py +95 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/_pixelstats.py +14 -65
- dataeval-1.2.0rc1/src/dataeval/core/_calculators/_redundancystats.py +48 -0
- dataeval-1.2.0rc1/src/dataeval/core/_calculators/_video_base.py +59 -0
- dataeval-1.2.0rc1/src/dataeval/core/_calculators/_video_register.py +12 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/_visualstats.py +13 -36
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_clusterer.py +1 -1
- dataeval-1.2.0rc1/src/dataeval/core/_combination_scores.py +850 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_completeness.py +11 -4
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_compute_ratios.py +189 -80
- dataeval-1.2.0rc1/src/dataeval/core/_compute_stats.py +362 -0
- dataeval-1.2.0rc1/src/dataeval/core/_concentration.py +465 -0
- dataeval-1.2.0rc1/src/dataeval/core/_consensus.py +239 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_coverage.py +5 -6
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_divergence.py +1 -1
- dataeval-1.2.0rc1/src/dataeval/core/_effective_n.py +68 -0
- dataeval-1.2.0rc1/src/dataeval/core/_ego_stats.py +1414 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_feature_distance.py +6 -12
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_hash.py +1 -1
- dataeval-1.2.0rc1/src/dataeval/core/_hash_index.py +938 -0
- dataeval-1.1.5/src/dataeval/core/_compute_stats.py → dataeval-1.2.0rc1/src/dataeval/core/_image_stats.py +463 -426
- dataeval-1.2.0rc1/src/dataeval/core/_label_cooccurrence.py +404 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_label_errors.py +1 -1
- dataeval-1.2.0rc1/src/dataeval/core/_label_parity.py +304 -0
- dataeval-1.2.0rc1/src/dataeval/core/_label_stats.py +509 -0
- dataeval-1.2.0rc1/src/dataeval/core/_local_panels.py +195 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_metadata_insights.py +21 -6
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_mst.py +88 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_mutual_info.py +89 -99
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_nullmodel.py +2 -3
- dataeval-1.2.0rc1/src/dataeval/core/_parity.py +425 -0
- dataeval-1.2.0rc1/src/dataeval/core/_persistence.py +431 -0
- dataeval-1.2.0rc1/src/dataeval/core/_rao_scott.py +428 -0
- dataeval-1.2.0rc1/src/dataeval/core/_sequence_match.py +463 -0
- dataeval-1.2.0rc1/src/dataeval/core/_sequence_residuals.py +563 -0
- dataeval-1.2.0rc1/src/dataeval/core/_temporal.py +200 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_track_stats.py +250 -82
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_uap.py +1 -1
- dataeval-1.2.0rc1/src/dataeval/core/_video_runner.py +243 -0
- dataeval-1.2.0rc1/src/dataeval/core/_video_stats.py +379 -0
- dataeval-1.2.0rc1/src/dataeval/data/__init__.py +94 -0
- dataeval-1.2.0rc1/src/dataeval/data/_camera_travel.py +198 -0
- dataeval-1.2.0rc1/src/dataeval/data/_chain.py +643 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_classbalance.py +23 -9
- dataeval-1.2.0rc1/src/dataeval/data/_classfilter.py +36 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_crop.py +10 -6
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_crops.py +366 -123
- dataeval-1.2.0rc1/src/dataeval/data/_filter.py +368 -0
- dataeval-1.2.0rc1/src/dataeval/data/_frames.py +1302 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_geometry.py +2 -1
- dataeval-1.2.0rc1/src/dataeval/data/_indices.py +232 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_invalidates.py +1 -1
- dataeval-1.2.0rc1/src/dataeval/data/_label_selectors.py +272 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_limit.py +3 -0
- dataeval-1.2.0rc1/src/dataeval/data/_locate.py +1363 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_merge.py +50 -2
- dataeval-1.2.0rc1/src/dataeval/data/_planners.py +205 -0
- dataeval-1.2.0rc1/src/dataeval/data/_quality_gate.py +233 -0
- dataeval-1.2.0rc1/src/dataeval/data/_reads.py +71 -0
- dataeval-1.2.0rc1/src/dataeval/data/_relabel.py +493 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_resize.py +16 -12
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_reverse.py +3 -0
- dataeval-1.2.0rc1/src/dataeval/data/_segments.py +409 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_selectchannels.py +8 -4
- dataeval-1.2.0rc1/src/dataeval/data/_selectors.py +1134 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_shuffle.py +4 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_split.py +245 -29
- dataeval-1.2.0rc1/src/dataeval/data/_stitch.py +612 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_torchvision.py +9 -4
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_tracks.py +9 -3
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_unzip.py +23 -3
- dataeval-1.2.0rc1/src/dataeval/data/_video_proxies.py +271 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/data/_view.py +154 -17
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/exceptions.py +8 -4
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/extractors/__init__.py +2 -2
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/extractors/_bovw.py +23 -5
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/extractors/_flatten.py +10 -3
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/extractors/_geometry.py +1 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/extractors/_onnx.py +6 -2
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/extractors/_scores.py +1 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/extractors/_torch.py +85 -48
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/extractors/_uncertainty.py +13 -217
- dataeval-1.2.0rc1/src/dataeval/extractors/_videotorch.py +541 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/flags.py +176 -2
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/models/_backends.py +3 -3
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/models/_input.py +3 -2
- dataeval-1.2.0rc1/src/dataeval/performance/__init__.py +20 -0
- dataeval-1.2.0rc1/src/dataeval/performance/_feasibility.py +982 -0
- dataeval-1.2.0rc1/src/dataeval/performance/_feasibility_classification.py +863 -0
- dataeval-1.2.0rc1/src/dataeval/performance/_feasibility_constants.py +24 -0
- dataeval-1.2.0rc1/src/dataeval/performance/_feasibility_decide.py +471 -0
- dataeval-1.2.0rc1/src/dataeval/performance/_feasibility_evidence.py +191 -0
- dataeval-1.2.0rc1/src/dataeval/performance/_feasibility_target.py +262 -0
- dataeval-1.2.0rc1/src/dataeval/performance/_feasibility_units.py +181 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/performance/_output.py +2 -3
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/performance/_sufficiency.py +3 -16
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/performance/schedules.py +1 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/protocols.py +242 -20
- dataeval-1.2.0rc1/src/dataeval/quality/_duplicates.py +5000 -0
- dataeval-1.2.0rc1/src/dataeval/quality/_factor_stats.py +165 -0
- dataeval-1.2.0rc1/src/dataeval/quality/_outliers.py +3818 -0
- dataeval-1.2.0rc1/src/dataeval/quality/_outliers_neighbors.py +234 -0
- dataeval-1.2.0rc1/src/dataeval/quality/_outliers_rows.py +461 -0
- dataeval-1.2.0rc1/src/dataeval/quality/_outliers_scores.py +555 -0
- dataeval-1.2.0rc1/src/dataeval/quality/_shared.py +417 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/scope/_coverage.py +102 -29
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/scope/_prioritize.py +120 -43
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/scope/_representation.py +62 -15
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/__init__.py +2 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/_base.py +230 -30
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/_chunk.py +10 -7
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/_domain_classifier.py +66 -16
- dataeval-1.2.0rc1/src/dataeval/shift/_drift/_domain_spec.py +249 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/_kneighbors.py +48 -13
- dataeval-1.2.0rc1/src/dataeval/shift/_drift/_known_domains.py +845 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/_mmd.py +46 -12
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/_reconstruction.py +10 -10
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/_univariate.py +76 -76
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/_wasserstein.py +53 -16
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_ood/_base.py +2 -2
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_ood/_domain_classifier.py +59 -12
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_ood/_kneighbors.py +55 -9
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_ood/_reconstruction.py +12 -12
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_shared/_domain_classifier.py +10 -4
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_shared/_reconstruction.py +2 -3
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/update_strategies.py +1 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/types/__init__.py +44 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/types/_config.py +13 -4
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/types/_evaluator.py +58 -16
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/types/_execution.py +1 -2
- dataeval-1.2.0rc1/src/dataeval/types/_factors.py +1727 -0
- dataeval-1.2.0rc1/src/dataeval/types/_index.py +317 -0
- dataeval-1.2.0rc1/src/dataeval/types/_known_factors.py +101 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/types/_output.py +113 -23
- dataeval-1.2.0rc1/src/dataeval/types/_removal.py +173 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/types/_schema.py +5 -6
- dataeval-1.2.0rc1/src/dataeval/types/_target.py +205 -0
- dataeval-1.2.0rc1/src/dataeval/types/_task.py +105 -0
- dataeval-1.2.0rc1/src/dataeval/types/_track.py +89 -0
- dataeval-1.2.0rc1/src/dataeval/utils/__init__.py +27 -0
- dataeval-1.2.0rc1/src/dataeval/utils/_array.py +331 -0
- dataeval-1.2.0rc1/src/dataeval/utils/_internal.py +181 -0
- dataeval-1.2.0rc1/src/dataeval/utils/_mask.py +91 -0
- dataeval-1.2.0rc1/src/dataeval/utils/_merge.py +462 -0
- dataeval-1.1.5/src/dataeval/utils/_validate.py → dataeval-1.2.0rc1/src/dataeval/utils/data.py +239 -45
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/utils/losses.py +1 -1
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/utils/preprocessing.py +28 -96
- dataeval-1.2.0rc1/src/dataeval/utils/scipy/__init__.py +5 -0
- dataeval-1.2.0rc1/src/dataeval/utils/scipy/stats.py +429 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/utils/thresholds.py +3 -4
- dataeval-1.1.5/src/dataeval/_metadata/_aggregate.py +0 -230
- dataeval-1.1.5/src/dataeval/_metadata/_deprecated.py +0 -392
- dataeval-1.1.5/src/dataeval/_metadata/_encoding.py +0 -297
- dataeval-1.1.5/src/dataeval/_metadata/_entry_legacy.py +0 -341
- dataeval-1.1.5/src/dataeval/_metadata/_input.py +0 -133
- dataeval-1.1.5/src/dataeval/_metadata/_structurers/_dataset.py +0 -92
- dataeval-1.1.5/src/dataeval/_metadata/_structurers/_instances.py +0 -60
- dataeval-1.1.5/src/dataeval/_metadata/_structurers/_source_index.py +0 -145
- dataeval-1.1.5/src/dataeval/bias/_parity.py +0 -284
- dataeval-1.1.5/src/dataeval/core/_label_parity.py +0 -215
- dataeval-1.1.5/src/dataeval/core/_label_stats.py +0 -212
- dataeval-1.1.5/src/dataeval/core/_parity.py +0 -188
- dataeval-1.1.5/src/dataeval/data/__init__.py +0 -42
- dataeval-1.1.5/src/dataeval/data/_classfilter.py +0 -71
- dataeval-1.1.5/src/dataeval/data/_indices.py +0 -24
- dataeval-1.1.5/src/dataeval/data/_relabel.py +0 -397
- dataeval-1.1.5/src/dataeval/performance/__init__.py +0 -11
- dataeval-1.1.5/src/dataeval/quality/_duplicates.py +0 -1484
- dataeval-1.1.5/src/dataeval/quality/_outliers.py +0 -1800
- dataeval-1.1.5/src/dataeval/quality/_shared.py +0 -199
- dataeval-1.1.5/src/dataeval/selection/__init__.py +0 -71
- dataeval-1.1.5/src/dataeval/types/_factors.py +0 -632
- dataeval-1.1.5/src/dataeval/types/_index.py +0 -92
- dataeval-1.1.5/src/dataeval/types/_track.py +0 -40
- dataeval-1.1.5/src/dataeval/utils/__init__.py +0 -19
- dataeval-1.1.5/src/dataeval/utils/_internal.py +0 -973
- dataeval-1.1.5/src/dataeval/utils/data.py +0 -34
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/LICENSE +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_block.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_gather.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_ordering.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/_metadata/_structurers/_propagation.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/bias/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/config.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/_register.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_calculators/_registry.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_diversity.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_fast_hdbscan/_cluster_trees.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_fast_hdbscan/_disjoint_set.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_fast_hdbscan/_mst.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_label_alignment.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_label_coverage.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_label_reconciliation.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_ontology_validation.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/core/_rank.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/models/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/models/_metadata.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/models/_predictors.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/performance/_aggregator.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/py.typed +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/quality/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/scope/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_drift/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_ood/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_shared/__init__.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/shift/_shared/_kneighbors.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/types/_array.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/types/_ontology.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/utils/models.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/utils/onnx.py +0 -0
- {dataeval-1.1.5 → dataeval-1.2.0rc1}/src/dataeval/utils/training.py +0 -0
|
@@ -12,6 +12,10 @@ docs/build/
|
|
|
12
12
|
docs/source/reference/autoapi
|
|
13
13
|
docs/source/tutorials/notebooks/checkpoints/
|
|
14
14
|
|
|
15
|
+
# Intersphinx inventories pre-fetched by docs/fetch-intersphinx-inventories.py
|
|
16
|
+
# (cached in CI, keyed on docs/source/conf.py)
|
|
17
|
+
docs/source/_intersphinx/
|
|
18
|
+
|
|
15
19
|
# Jupyter cache is stored in orphan branches (docs-artifacts/<branch>)
|
|
16
20
|
# See scripts/README.md for details
|
|
17
21
|
.jupyter_cache/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: dataeval
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0rc1
|
|
4
4
|
Summary: DataEval provides a simple interface to characterize image data and its impact on model performance across classification and object-detection tasks
|
|
5
5
|
Project-URL: Homepage, https://dataeval.ai/
|
|
6
6
|
Project-URL: Repository, https://github.com/aria-ml/dataeval/
|
|
@@ -13,13 +13,12 @@ Classifier: Development Status :: 5 - Production/Stable
|
|
|
13
13
|
Classifier: Intended Audience :: Science/Research
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.11
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.14
|
|
21
20
|
Classifier: Topic :: Scientific/Engineering
|
|
22
|
-
Requires-Python: >=3.
|
|
21
|
+
Requires-Python: >=3.11
|
|
23
22
|
Requires-Dist: lightgbm>=4
|
|
24
23
|
Requires-Dist: maite>=0.9.4
|
|
25
24
|
Requires-Dist: numba>=0.61.0
|
|
@@ -48,13 +47,11 @@ Requires-Dist: onnx>=1.14.0; extra == 'onnx'
|
|
|
48
47
|
Requires-Dist: onnxruntime>=1.17; extra == 'onnx'
|
|
49
48
|
Provides-Extra: onnx-cu126
|
|
50
49
|
Requires-Dist: onnx>=1.14.0; extra == 'onnx-cu126'
|
|
51
|
-
Requires-Dist: onnxruntime-gpu<1.
|
|
52
|
-
Requires-Dist: onnxruntime-gpu<1.27,>=1.17; (python_version >= '3.11' and extra != 'onnx-cu130') and extra == 'onnx-cu126'
|
|
50
|
+
Requires-Dist: onnxruntime-gpu<1.27,>=1.17; (extra != 'onnx-cu130') and extra == 'onnx-cu126'
|
|
53
51
|
Requires-Dist: onnxruntime-gpu<1.27,>=1.24; (python_version >= '3.14' and extra != 'onnx-cu130') and extra == 'onnx-cu126'
|
|
54
52
|
Provides-Extra: onnx-cu130
|
|
55
53
|
Requires-Dist: onnx>=1.14.0; extra == 'onnx-cu130'
|
|
56
|
-
Requires-Dist: onnxruntime-gpu>=1.27; (
|
|
57
|
-
Requires-Dist: onnxruntime>=1.17; (python_version == '3.10' and extra != 'onnx-cu126') and extra == 'onnx-cu130'
|
|
54
|
+
Requires-Dist: onnxruntime-gpu>=1.27; (extra != 'onnx-cu126') and extra == 'onnx-cu130'
|
|
58
55
|
Provides-Extra: ontology
|
|
59
56
|
Requires-Dist: rdflib>=7.0; extra == 'ontology'
|
|
60
57
|
Provides-Extra: opencv
|
|
@@ -94,7 +91,7 @@ DataEval addresses the critical need underlying every AI model -- the data.
|
|
|
94
91
|
The difference between a great dataset and a poor dataset can have drastic
|
|
95
92
|
consequences on AI model performance. Data collected in the wild is noisy,
|
|
96
93
|
often imbalanced, and doesn't always cover the entire spectrum of conditions
|
|
97
|
-
|
|
94
|
+
needed for deployment. DataEval provides AI practitioners with a library of
|
|
98
95
|
rigorous, algorithm-backed metrics for performance estimation, bias analysis,
|
|
99
96
|
dataset cleaning and assessment, and data distribution shifts. Throughout
|
|
100
97
|
all stages of the machine learning lifecycle -- from initial data collection
|
|
@@ -107,22 +104,22 @@ communities.
|
|
|
107
104
|
|
|
108
105
|
<!-- end needs -->
|
|
109
106
|
|
|
110
|
-
### Target
|
|
107
|
+
### Target audience
|
|
111
108
|
|
|
112
109
|
<!-- start JATIC interop -->
|
|
113
110
|
|
|
114
111
|
DataEval is intended to help data scientists, developers, and T&E engineers
|
|
115
112
|
who want to evaluate and enhance their datasets for optimum performance. For
|
|
116
|
-
users of the
|
|
113
|
+
users of the JATIC product suite, DataEval has native interoperability when
|
|
117
114
|
using MAITE-compliant datasets and models.
|
|
118
115
|
|
|
119
116
|
<!-- end JATIC interop -->
|
|
120
117
|
|
|
121
118
|
---
|
|
122
119
|
|
|
123
|
-
## Getting
|
|
120
|
+
## Getting started
|
|
124
121
|
|
|
125
|
-
**Python versions:** 3.
|
|
122
|
+
**Python versions:** 3.11 - 3.14
|
|
126
123
|
|
|
127
124
|
Choose your preferred method of installation below or follow our
|
|
128
125
|
[installation guide](docs/source/getting-started/installation.md).
|
|
@@ -132,7 +129,7 @@ Choose your preferred method of installation below or follow our
|
|
|
132
129
|
- [Installing with conda/mamba](#installing-with-conda)
|
|
133
130
|
- [Installing from GitHub](#installing-from-github)
|
|
134
131
|
|
|
135
|
-
###
|
|
132
|
+
### Installing with pip
|
|
136
133
|
|
|
137
134
|
You can install DataEval directly from pypi.org using the following command.
|
|
138
135
|
|
|
@@ -170,7 +167,7 @@ pip install dataeval
|
|
|
170
167
|
> `--index-url` under pip, `--torch-backend` under `uv pip`, and use the extras
|
|
171
168
|
> only for source installs.
|
|
172
169
|
|
|
173
|
-
###
|
|
170
|
+
### torchvision (optional)
|
|
174
171
|
|
|
175
172
|
`torchvision` is not a DataEval dependency, and nothing imports it until you reach
|
|
176
173
|
for `TorchvisionTransform` — the escape hatch for running a torchvision v2
|
|
@@ -189,13 +186,13 @@ pip install torchvision --index-url https://download.pytorch.org/whl/cu130
|
|
|
189
186
|
> the same index — which is also why `pip install dataeval[cpu]` is the wrong way
|
|
190
187
|
> to obtain it.
|
|
191
188
|
|
|
192
|
-
###
|
|
189
|
+
### Installing with uv
|
|
193
190
|
|
|
194
191
|
```bash
|
|
195
192
|
uv pip install dataeval --torch-backend cpu # or cu126 / cu130 / auto
|
|
196
193
|
```
|
|
197
194
|
|
|
198
|
-
###
|
|
195
|
+
### Installing with conda
|
|
199
196
|
|
|
200
197
|
DataEval can be installed from conda-forge:
|
|
201
198
|
|
|
@@ -212,7 +209,7 @@ variant.
|
|
|
212
209
|
micromamba create -f environment.yml
|
|
213
210
|
```
|
|
214
211
|
|
|
215
|
-
###
|
|
212
|
+
### Installing from GitHub
|
|
216
213
|
|
|
217
214
|
To install DataEval from source locally on Ubuntu, pull the source down and
|
|
218
215
|
change to the DataEval project directory.
|
|
@@ -227,21 +224,7 @@ cd dataeval
|
|
|
227
224
|
> linting, type checking and docs tooling included. See
|
|
228
225
|
> [Development Setup](./CONTRIBUTING.md#development-setup) for the options it takes.
|
|
229
226
|
|
|
230
|
-
####
|
|
231
|
-
|
|
232
|
-
Install DataEval.
|
|
233
|
-
|
|
234
|
-
```bash
|
|
235
|
-
poetry install
|
|
236
|
-
```
|
|
237
|
-
|
|
238
|
-
Enable Poetry's virtual environment.
|
|
239
|
-
|
|
240
|
-
```bash
|
|
241
|
-
poetry env activate
|
|
242
|
-
```
|
|
243
|
-
|
|
244
|
-
#### **Using uv**
|
|
227
|
+
#### Using uv
|
|
245
228
|
|
|
246
229
|
Install DataEval with dependencies for development.
|
|
247
230
|
|
|
@@ -262,7 +245,7 @@ DataEval has two input paths depending on which part of the library you are usin
|
|
|
262
245
|
**`dataeval.core`** provides stateless functions that operate directly on NumPy
|
|
263
246
|
arrays — embeddings, labels, image hashes, and statistics. No dataset object is
|
|
264
247
|
required. Call these functions with arrays and get results back directly. Examples
|
|
265
|
-
include `
|
|
248
|
+
include `image_stats`, `label_errors`, `divergence_mst`, and `ber_knn`.
|
|
266
249
|
|
|
267
250
|
**`dataeval.quality`, `dataeval.bias`, `dataeval.shift`, and `dataeval.performance`**
|
|
268
251
|
provide stateful evaluator classes (`Duplicates`, `Outliers`,
|
|
@@ -364,7 +347,7 @@ class MyObjectDetectionDataset(od.Dataset):
|
|
|
364
347
|
|
|
365
348
|
self.metadata = mp.DatasetMetadata(
|
|
366
349
|
id="my_object_detection_dataset",
|
|
367
|
-
index2label={i: f"class_{i}" for i in
|
|
350
|
+
index2label={i: f"class_{i}" for i in sorted({lab for labs in labels for lab in labs})}, # example mapping
|
|
368
351
|
)
|
|
369
352
|
|
|
370
353
|
def __len__(self) -> int:
|
|
@@ -447,19 +430,19 @@ from dataeval.quality import Duplicates
|
|
|
447
430
|
extractor = TorchExtractor(Flatten())
|
|
448
431
|
|
|
449
432
|
# Find near-duplicates using only embedding-based clustering.
|
|
450
|
-
# An aggressive
|
|
433
|
+
# An aggressive cluster_sensitivity of 1.5 should produce detections
|
|
451
434
|
# of near duplicates even with a simple Flatten extractor.
|
|
452
435
|
evaluator = Duplicates(
|
|
453
436
|
flags=ImageStats.NONE,
|
|
454
437
|
cluster_algorithm="hdbscan",
|
|
455
|
-
|
|
438
|
+
cluster_sensitivity=1.5,
|
|
456
439
|
extractor=extractor,
|
|
457
440
|
batch_size=64,
|
|
458
441
|
)
|
|
459
442
|
result = evaluator.evaluate(dataset)
|
|
460
443
|
|
|
461
|
-
# Near duplicates are grouped into sets of indices
|
|
462
|
-
#
|
|
444
|
+
# Near duplicates are grouped into sets of indices whose pairwise distances
|
|
445
|
+
# fall below `cluster_sensitivity * cluster_std` in embedding space.
|
|
463
446
|
print(result)
|
|
464
447
|
```
|
|
465
448
|
|
|
@@ -488,22 +471,22 @@ guide to find the right evaluator for your situation.
|
|
|
488
471
|
|
|
489
472
|
Know which tool to use, then check out [What data does each tool need?](./docs/source/getting-started/input-requirements.md)
|
|
490
473
|
for a quick-reference table of every algorithm's inputs, and the
|
|
491
|
-
[Functional
|
|
474
|
+
[Functional overview](./docs/source/reference/FunctionalOverview.md) for task applicability.
|
|
492
475
|
|
|
493
476
|
Want to just explore the documentation? The [Where to go next](./docs/source/getting-started/where-to-go-next.md)
|
|
494
477
|
page allows you to jump around between the different areas of the documentation with small summaries of what each page covers.
|
|
495
478
|
|
|
496
479
|
---
|
|
497
480
|
|
|
498
|
-
## Contact
|
|
481
|
+
## Contact us
|
|
499
482
|
|
|
500
483
|
If you have any questions, feel free to reach out to [us](mailto:dataeval@ariacoustics.com)!
|
|
501
484
|
|
|
502
|
-
##
|
|
485
|
+
## Acknowledgment
|
|
503
486
|
|
|
504
|
-
### CDAO
|
|
487
|
+
### CDAO funding acknowledgment
|
|
505
488
|
|
|
506
|
-
<!-- start
|
|
489
|
+
<!-- start acknowledgment -->
|
|
507
490
|
|
|
508
491
|
This material is based upon work supported by the Chief Digital and Artificial
|
|
509
492
|
Intelligence Office under Contract No. W519TC-23-9-2033. The views and
|
|
@@ -511,4 +494,4 @@ conclusions contained herein are those of the author(s) and should not be
|
|
|
511
494
|
interpreted as necessarily representing the official policies or endorsements,
|
|
512
495
|
either expressed or implied, of the U.S. Government.
|
|
513
496
|
|
|
514
|
-
<!-- end
|
|
497
|
+
<!-- end acknowledgment -->
|
|
@@ -31,7 +31,7 @@ DataEval addresses the critical need underlying every AI model -- the data.
|
|
|
31
31
|
The difference between a great dataset and a poor dataset can have drastic
|
|
32
32
|
consequences on AI model performance. Data collected in the wild is noisy,
|
|
33
33
|
often imbalanced, and doesn't always cover the entire spectrum of conditions
|
|
34
|
-
|
|
34
|
+
needed for deployment. DataEval provides AI practitioners with a library of
|
|
35
35
|
rigorous, algorithm-backed metrics for performance estimation, bias analysis,
|
|
36
36
|
dataset cleaning and assessment, and data distribution shifts. Throughout
|
|
37
37
|
all stages of the machine learning lifecycle -- from initial data collection
|
|
@@ -44,22 +44,22 @@ communities.
|
|
|
44
44
|
|
|
45
45
|
<!-- end needs -->
|
|
46
46
|
|
|
47
|
-
### Target
|
|
47
|
+
### Target audience
|
|
48
48
|
|
|
49
49
|
<!-- start JATIC interop -->
|
|
50
50
|
|
|
51
51
|
DataEval is intended to help data scientists, developers, and T&E engineers
|
|
52
52
|
who want to evaluate and enhance their datasets for optimum performance. For
|
|
53
|
-
users of the
|
|
53
|
+
users of the JATIC product suite, DataEval has native interoperability when
|
|
54
54
|
using MAITE-compliant datasets and models.
|
|
55
55
|
|
|
56
56
|
<!-- end JATIC interop -->
|
|
57
57
|
|
|
58
58
|
---
|
|
59
59
|
|
|
60
|
-
## Getting
|
|
60
|
+
## Getting started
|
|
61
61
|
|
|
62
|
-
**Python versions:** 3.
|
|
62
|
+
**Python versions:** 3.11 - 3.14
|
|
63
63
|
|
|
64
64
|
Choose your preferred method of installation below or follow our
|
|
65
65
|
[installation guide](docs/source/getting-started/installation.md).
|
|
@@ -69,7 +69,7 @@ Choose your preferred method of installation below or follow our
|
|
|
69
69
|
- [Installing with conda/mamba](#installing-with-conda)
|
|
70
70
|
- [Installing from GitHub](#installing-from-github)
|
|
71
71
|
|
|
72
|
-
###
|
|
72
|
+
### Installing with pip
|
|
73
73
|
|
|
74
74
|
You can install DataEval directly from pypi.org using the following command.
|
|
75
75
|
|
|
@@ -107,7 +107,7 @@ pip install dataeval
|
|
|
107
107
|
> `--index-url` under pip, `--torch-backend` under `uv pip`, and use the extras
|
|
108
108
|
> only for source installs.
|
|
109
109
|
|
|
110
|
-
###
|
|
110
|
+
### torchvision (optional)
|
|
111
111
|
|
|
112
112
|
`torchvision` is not a DataEval dependency, and nothing imports it until you reach
|
|
113
113
|
for `TorchvisionTransform` — the escape hatch for running a torchvision v2
|
|
@@ -126,13 +126,13 @@ pip install torchvision --index-url https://download.pytorch.org/whl/cu130
|
|
|
126
126
|
> the same index — which is also why `pip install dataeval[cpu]` is the wrong way
|
|
127
127
|
> to obtain it.
|
|
128
128
|
|
|
129
|
-
###
|
|
129
|
+
### Installing with uv
|
|
130
130
|
|
|
131
131
|
```bash
|
|
132
132
|
uv pip install dataeval --torch-backend cpu # or cu126 / cu130 / auto
|
|
133
133
|
```
|
|
134
134
|
|
|
135
|
-
###
|
|
135
|
+
### Installing with conda
|
|
136
136
|
|
|
137
137
|
DataEval can be installed from conda-forge:
|
|
138
138
|
|
|
@@ -149,7 +149,7 @@ variant.
|
|
|
149
149
|
micromamba create -f environment.yml
|
|
150
150
|
```
|
|
151
151
|
|
|
152
|
-
###
|
|
152
|
+
### Installing from GitHub
|
|
153
153
|
|
|
154
154
|
To install DataEval from source locally on Ubuntu, pull the source down and
|
|
155
155
|
change to the DataEval project directory.
|
|
@@ -164,21 +164,7 @@ cd dataeval
|
|
|
164
164
|
> linting, type checking and docs tooling included. See
|
|
165
165
|
> [Development Setup](./CONTRIBUTING.md#development-setup) for the options it takes.
|
|
166
166
|
|
|
167
|
-
####
|
|
168
|
-
|
|
169
|
-
Install DataEval.
|
|
170
|
-
|
|
171
|
-
```bash
|
|
172
|
-
poetry install
|
|
173
|
-
```
|
|
174
|
-
|
|
175
|
-
Enable Poetry's virtual environment.
|
|
176
|
-
|
|
177
|
-
```bash
|
|
178
|
-
poetry env activate
|
|
179
|
-
```
|
|
180
|
-
|
|
181
|
-
#### **Using uv**
|
|
167
|
+
#### Using uv
|
|
182
168
|
|
|
183
169
|
Install DataEval with dependencies for development.
|
|
184
170
|
|
|
@@ -199,7 +185,7 @@ DataEval has two input paths depending on which part of the library you are usin
|
|
|
199
185
|
**`dataeval.core`** provides stateless functions that operate directly on NumPy
|
|
200
186
|
arrays — embeddings, labels, image hashes, and statistics. No dataset object is
|
|
201
187
|
required. Call these functions with arrays and get results back directly. Examples
|
|
202
|
-
include `
|
|
188
|
+
include `image_stats`, `label_errors`, `divergence_mst`, and `ber_knn`.
|
|
203
189
|
|
|
204
190
|
**`dataeval.quality`, `dataeval.bias`, `dataeval.shift`, and `dataeval.performance`**
|
|
205
191
|
provide stateful evaluator classes (`Duplicates`, `Outliers`,
|
|
@@ -301,7 +287,7 @@ class MyObjectDetectionDataset(od.Dataset):
|
|
|
301
287
|
|
|
302
288
|
self.metadata = mp.DatasetMetadata(
|
|
303
289
|
id="my_object_detection_dataset",
|
|
304
|
-
index2label={i: f"class_{i}" for i in
|
|
290
|
+
index2label={i: f"class_{i}" for i in sorted({lab for labs in labels for lab in labs})}, # example mapping
|
|
305
291
|
)
|
|
306
292
|
|
|
307
293
|
def __len__(self) -> int:
|
|
@@ -384,19 +370,19 @@ from dataeval.quality import Duplicates
|
|
|
384
370
|
extractor = TorchExtractor(Flatten())
|
|
385
371
|
|
|
386
372
|
# Find near-duplicates using only embedding-based clustering.
|
|
387
|
-
# An aggressive
|
|
373
|
+
# An aggressive cluster_sensitivity of 1.5 should produce detections
|
|
388
374
|
# of near duplicates even with a simple Flatten extractor.
|
|
389
375
|
evaluator = Duplicates(
|
|
390
376
|
flags=ImageStats.NONE,
|
|
391
377
|
cluster_algorithm="hdbscan",
|
|
392
|
-
|
|
378
|
+
cluster_sensitivity=1.5,
|
|
393
379
|
extractor=extractor,
|
|
394
380
|
batch_size=64,
|
|
395
381
|
)
|
|
396
382
|
result = evaluator.evaluate(dataset)
|
|
397
383
|
|
|
398
|
-
# Near duplicates are grouped into sets of indices
|
|
399
|
-
#
|
|
384
|
+
# Near duplicates are grouped into sets of indices whose pairwise distances
|
|
385
|
+
# fall below `cluster_sensitivity * cluster_std` in embedding space.
|
|
400
386
|
print(result)
|
|
401
387
|
```
|
|
402
388
|
|
|
@@ -425,22 +411,22 @@ guide to find the right evaluator for your situation.
|
|
|
425
411
|
|
|
426
412
|
Know which tool to use, then check out [What data does each tool need?](./docs/source/getting-started/input-requirements.md)
|
|
427
413
|
for a quick-reference table of every algorithm's inputs, and the
|
|
428
|
-
[Functional
|
|
414
|
+
[Functional overview](./docs/source/reference/FunctionalOverview.md) for task applicability.
|
|
429
415
|
|
|
430
416
|
Want to just explore the documentation? The [Where to go next](./docs/source/getting-started/where-to-go-next.md)
|
|
431
417
|
page allows you to jump around between the different areas of the documentation with small summaries of what each page covers.
|
|
432
418
|
|
|
433
419
|
---
|
|
434
420
|
|
|
435
|
-
## Contact
|
|
421
|
+
## Contact us
|
|
436
422
|
|
|
437
423
|
If you have any questions, feel free to reach out to [us](mailto:dataeval@ariacoustics.com)!
|
|
438
424
|
|
|
439
|
-
##
|
|
425
|
+
## Acknowledgment
|
|
440
426
|
|
|
441
|
-
### CDAO
|
|
427
|
+
### CDAO funding acknowledgment
|
|
442
428
|
|
|
443
|
-
<!-- start
|
|
429
|
+
<!-- start acknowledgment -->
|
|
444
430
|
|
|
445
431
|
This material is based upon work supported by the Chief Digital and Artificial
|
|
446
432
|
Intelligence Office under Contract No. W519TC-23-9-2033. The views and
|
|
@@ -448,4 +434,4 @@ conclusions contained herein are those of the author(s) and should not be
|
|
|
448
434
|
interpreted as necessarily representing the official policies or endorsements,
|
|
449
435
|
either expressed or implied, of the U.S. Government.
|
|
450
436
|
|
|
451
|
-
<!-- end
|
|
437
|
+
<!-- end acknowledgment -->
|
|
@@ -12,7 +12,7 @@ authors = [
|
|
|
12
12
|
{ name = "Ryan Wood", email = "ryan.wood@ariacoustics.com" },
|
|
13
13
|
{ name = "Scott Swan", email = "scott.swan@ariacoustics.com" },
|
|
14
14
|
]
|
|
15
|
-
requires-python = ">=3.
|
|
15
|
+
requires-python = ">=3.11"
|
|
16
16
|
readme = "README.md"
|
|
17
17
|
license = "MIT"
|
|
18
18
|
maintainers = [
|
|
@@ -23,7 +23,6 @@ classifiers = [
|
|
|
23
23
|
"Operating System :: OS Independent",
|
|
24
24
|
"Intended Audience :: Science/Research",
|
|
25
25
|
"Programming Language :: Python :: 3 :: Only",
|
|
26
|
-
"Programming Language :: Python :: 3.10",
|
|
27
26
|
"Programming Language :: Python :: 3.11",
|
|
28
27
|
"Programming Language :: Python :: 3.12",
|
|
29
28
|
"Programming Language :: Python :: 3.13",
|
|
@@ -53,18 +52,19 @@ litert = ["ai-edge-litert>=2.0; python_version <= '3.14'"]
|
|
|
53
52
|
opencv = ["opencv-python-headless>=4.8.0"]
|
|
54
53
|
onnx = [
|
|
55
54
|
"onnx>=1.14.0",
|
|
55
|
+
# "onnxmltools>=1.16", # required for LightGBM ONNX export
|
|
56
56
|
"onnxruntime>=1.17",
|
|
57
57
|
]
|
|
58
58
|
onnx-cu126 = [
|
|
59
59
|
"onnx>=1.14.0",
|
|
60
|
-
"
|
|
61
|
-
"onnxruntime-gpu>=1.17,<1.27;
|
|
60
|
+
# "onnxmltools>=1.16", # required for LightGBM ONNX export
|
|
61
|
+
"onnxruntime-gpu>=1.17,<1.27; extra != 'onnx-cu130'", # 1.27+ requires CUDA 13.0
|
|
62
62
|
"onnxruntime-gpu>=1.24,<1.27; python_version >= '3.14' and extra != 'onnx-cu130'", # 1.24+ for cp314 wheels
|
|
63
63
|
]
|
|
64
64
|
onnx-cu130 = [
|
|
65
65
|
"onnx>=1.14.0",
|
|
66
|
-
"
|
|
67
|
-
"onnxruntime-gpu>=1.27;
|
|
66
|
+
# "onnxmltools>=1.16", # required for LightGBM ONNX export
|
|
67
|
+
"onnxruntime-gpu>=1.27; extra != 'onnx-cu126'",
|
|
68
68
|
]
|
|
69
69
|
ontology = ["rdflib>=7.0"]
|
|
70
70
|
|
|
@@ -97,7 +97,6 @@ base = [
|
|
|
97
97
|
lock = [
|
|
98
98
|
{ include-group = "base" },
|
|
99
99
|
"pyproject2conda>=0.22",
|
|
100
|
-
"poetry==2.2.0; python_version<'3.14'",
|
|
101
100
|
]
|
|
102
101
|
lint = [
|
|
103
102
|
"ruff>=0.15",
|
|
@@ -109,34 +108,35 @@ docsync = [
|
|
|
109
108
|
]
|
|
110
109
|
test = [
|
|
111
110
|
"coverage[toml]>=7.6",
|
|
112
|
-
"
|
|
113
|
-
"pytest>=8.3",
|
|
111
|
+
"pytest>=7.3.1", # floor capped by the JATIC program standard; uv.lock resolves current
|
|
114
112
|
"pytest-cov>=6.1",
|
|
115
113
|
"pytest-xdist>=3.6.1",
|
|
116
114
|
]
|
|
117
115
|
test-onnx = [
|
|
118
116
|
{ include-group = "test" },
|
|
117
|
+
"onnxmltools>=1.16",
|
|
119
118
|
"onnxscript>=0.6.0",
|
|
120
119
|
]
|
|
121
120
|
verify = [
|
|
122
|
-
"pytest>=
|
|
121
|
+
"pytest>=7.3.1", # floor capped by the JATIC program standard; uv.lock resolves current
|
|
123
122
|
"pyyaml>=6.0",
|
|
124
123
|
]
|
|
125
124
|
type = [
|
|
126
125
|
"pyright[nodejs]>=1.1.400",
|
|
127
|
-
"
|
|
128
|
-
"
|
|
126
|
+
"requests>=2.31",
|
|
127
|
+
"pyyaml>=6.0",
|
|
129
128
|
]
|
|
130
129
|
docs = [
|
|
131
|
-
"numpy>=
|
|
130
|
+
"numpy>=1.24.2", # floor capped by the JATIC program standard; uv.lock resolves current
|
|
131
|
+
"av>=12.0.0",
|
|
132
132
|
"certifi>=2024.07.04",
|
|
133
|
-
"dataeval-plots[plotly]>=0.0.
|
|
133
|
+
"dataeval-plots[plotly]>=0.0.15",
|
|
134
134
|
"ipykernel>=6.26.0",
|
|
135
135
|
"ipywidgets>=8.1.1",
|
|
136
136
|
"jinja2>=3.1.6",
|
|
137
137
|
"jupyter-client>=8.6.0",
|
|
138
138
|
"jupyter-cache>=1.0",
|
|
139
|
-
"maite-datasets>=0.0.
|
|
139
|
+
"maite-datasets[hf]>=0.0.21",
|
|
140
140
|
"myst-nb>=1.0",
|
|
141
141
|
"opencv-python-headless>=4.8.0",
|
|
142
142
|
"pandas>=2.0.0",
|
|
@@ -154,22 +154,6 @@ docs = [
|
|
|
154
154
|
"jupytext>=1.19.1",
|
|
155
155
|
"ultralytics>=8.0.0",
|
|
156
156
|
]
|
|
157
|
-
security = [ # keep in sync with [tool.uv.constraint-dependencies]
|
|
158
|
-
"cryptography>=46.0.5", # CVE-2026-26007: Missing Subgroup Validation for SECT Curves
|
|
159
|
-
"filelock>=3.20.3", # GHSA-w853-jp5j-5j7f, GHSA-qmgc-5h2g-mvrw
|
|
160
|
-
"pillow>=12.1.1", # CVE-2026-25990: OOB write via PSD image
|
|
161
|
-
"protobuf>=6.33.5", # GHSA-7gcm-g887-7qv7
|
|
162
|
-
"setuptools>=82.0.0", # CVE-2026-23949: (jaraco_context) path traversal in tarball()
|
|
163
|
-
# CVE-2026-24049: (wheel) privilege escalation via unpack
|
|
164
|
-
# FIPS: opencv builds newer than 4.12.0.88 are not approved for FIPS-enabled
|
|
165
|
-
# hosts. All three distributions below ship the same `cv2` module and are
|
|
166
|
-
# interchangeable to a resolver, so capping only one leaves the others free to
|
|
167
|
-
# supply a newer cv2 (ultralytics pulls opencv-python). Cap all three.
|
|
168
|
-
# Do not raise this ceiling without a compliance review.
|
|
169
|
-
"opencv-python-headless<=4.12.0.88",
|
|
170
|
-
"opencv-python<=4.12.0.88",
|
|
171
|
-
"opencv-contrib-python<=4.12.0.88",
|
|
172
|
-
]
|
|
173
157
|
dev = [
|
|
174
158
|
{ include-group = "base" },
|
|
175
159
|
{ include-group = "lint" },
|
|
@@ -179,7 +163,7 @@ dev = [
|
|
|
179
163
|
"nox>=2025.5.1",
|
|
180
164
|
"nox-uv>=0.6.2",
|
|
181
165
|
"pip>=25.1",
|
|
182
|
-
"jupyter",
|
|
166
|
+
"jupyter>=1.0",
|
|
183
167
|
]
|
|
184
168
|
|
|
185
169
|
[tool.uv]
|
|
@@ -197,8 +181,12 @@ conflicts = [
|
|
|
197
181
|
]
|
|
198
182
|
constraint-dependencies = [
|
|
199
183
|
"cryptography>=46.0.5", # CVE-2026-26007: Missing Subgroup Validation for SECT Curves
|
|
200
|
-
"filelock>=
|
|
201
|
-
|
|
184
|
+
"filelock>=4.2.1", # CVE-2026-112940: (GHSA-v4x8-3wr5-3wq8) ReadWriteLock lets SQLite reopen files by name
|
|
185
|
+
# GHSA-w853-jp5j-5j7f, GHSA-qmgc-5h2g-mvrw (fixed earlier, in 3.20.3)
|
|
186
|
+
"pillow>=13.0.0", # Pillow 13.0.0 security release (decoder/encoder OOB reads and writes, DoS):
|
|
187
|
+
# CVE-2026-112395, CVE-2026-112397, CVE-2026-112400,
|
|
188
|
+
# CVE-2026-112403, CVE-2026-112404, CVE-2026-112895,
|
|
189
|
+
# CVE-2026-112896, CVE-2026-112897, CVE-2026-112899
|
|
202
190
|
"protobuf>=6.33.5", # GHSA-7gcm-g887-7qv7
|
|
203
191
|
"setuptools>=82.0.0", # CVE-2026-23949: (jaraco_context) path traversal in tarball()
|
|
204
192
|
# CVE-2026-24049: (wheel) privilege escalation via unpack
|
|
@@ -242,17 +230,12 @@ torchvision = [
|
|
|
242
230
|
[tool.uv.extra-build-dependencies]
|
|
243
231
|
numba = ["tbb>=2021.6"]
|
|
244
232
|
|
|
245
|
-
[tool.
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
priority = "supplemental"
|
|
252
|
-
|
|
253
|
-
[tool.poetry.dependencies]
|
|
254
|
-
torch = { version = ">=2.2.0", source = "pytorch-cpu" }
|
|
255
|
-
torchvision = { version = ">=0.17.0", source = "pytorch-cpu" }
|
|
233
|
+
[tool.hatch.build]
|
|
234
|
+
# CI keeps the uv cache in the project directory. `uv sync` extracts packages into it while hatchling walks the project to build
|
|
235
|
+
# dataeval, and a directory that disappears mid-walk fails the build with FileNotFoundError. Without skip-excluded-dirs,
|
|
236
|
+
# hatchling filters excluded paths only after it has walked into them.
|
|
237
|
+
skip-excluded-dirs = true
|
|
238
|
+
exclude = [".uv-cache"]
|
|
256
239
|
|
|
257
240
|
[tool.hatch.build.targets.sdist]
|
|
258
241
|
include = ["src/dataeval"]
|
|
@@ -269,12 +252,6 @@ source = "vcs"
|
|
|
269
252
|
[tool.hatch.build.hooks.vcs]
|
|
270
253
|
version-file = "src/dataeval/_version.py"
|
|
271
254
|
|
|
272
|
-
[tool.poetry-dynamic-versioning]
|
|
273
|
-
enable = true
|
|
274
|
-
vcs = "git"
|
|
275
|
-
style = "pep440"
|
|
276
|
-
pattern = "^v?(?P<base>\\d+\\.\\d+\\.\\d+)"
|
|
277
|
-
|
|
278
255
|
[tool.pyproject2conda]
|
|
279
256
|
# maite is only published on conda-forge; the defaults channel cannot resolve it.
|
|
280
257
|
channels = ["conda-forge"]
|
|
@@ -301,8 +278,7 @@ reportMissingImports = false
|
|
|
301
278
|
[tool.pytest.ini_options]
|
|
302
279
|
testpaths = ["tests"]
|
|
303
280
|
filterwarnings = [
|
|
304
|
-
"ignore:The default value of
|
|
305
|
-
"ignore:Clustering metrics expect discrete values but received continuous values:UserWarning",
|
|
281
|
+
"ignore:The default value of hash_radius will change:FutureWarning",
|
|
306
282
|
]
|
|
307
283
|
addopts = [
|
|
308
284
|
"--pythonwarnings=ignore::DeprecationWarning",
|
|
@@ -314,7 +290,9 @@ markers = [
|
|
|
314
290
|
"required: marks tests for required features",
|
|
315
291
|
"optional: marks tests for optional features",
|
|
316
292
|
"cuda: marks tests that require cuda",
|
|
293
|
+
"ffmpeg: marks integration tests that run against ffmpeg-generated video fixtures",
|
|
317
294
|
"performance: marks verification tests that time workloads or measure memory",
|
|
295
|
+
"infrastructure: marks tests of the documentation and repository tooling, which need the source checkout and git",
|
|
318
296
|
]
|
|
319
297
|
|
|
320
298
|
[tool.coverage.run]
|
|
@@ -359,7 +337,7 @@ exclude = [
|
|
|
359
337
|
]
|
|
360
338
|
line-length = 120
|
|
361
339
|
indent-width = 4
|
|
362
|
-
target-version = "
|
|
340
|
+
target-version = "py311"
|
|
363
341
|
extend-include = ["*.ipynb"]
|
|
364
342
|
|
|
365
343
|
[tool.ruff.lint]
|
|
@@ -399,8 +377,11 @@ docstring-code-format = true
|
|
|
399
377
|
docstring-code-line-length = "dynamic"
|
|
400
378
|
|
|
401
379
|
[tool.typos.default]
|
|
380
|
+
locale = "en-us"
|
|
402
381
|
extend-ignore-re = [
|
|
403
|
-
"`[0-9a-fA-F]{7,40}`"
|
|
382
|
+
"`[0-9a-fA-F]{7,40}`",
|
|
383
|
+
"get_cluster_labelling_at_cut", # vendored fast_hdbscan API name
|
|
384
|
+
"MISPmicrosectime", # MISB ST 0604 timestamp identifier
|
|
404
385
|
]
|
|
405
386
|
|
|
406
387
|
[tool.typos.default.extend-words]
|
|
@@ -410,17 +391,20 @@ iy = "iy"
|
|
|
410
391
|
fpr = "fpr"
|
|
411
392
|
Pont = "Pont"
|
|
412
393
|
setp = "setp"
|
|
394
|
+
writeable = "writeable"
|
|
395
|
+
aeroplane = "aeroplane" # Pascal VOC class label
|
|
413
396
|
|
|
414
397
|
[tool.typos.files]
|
|
415
398
|
extend-exclude = [
|
|
416
399
|
"requirements.*.txt",
|
|
417
400
|
"uv.lock",
|
|
418
|
-
"
|
|
401
|
+
"*.svg", # generated figures: path data and random element ids
|
|
419
402
|
]
|
|
420
403
|
|
|
421
404
|
[tool.codespell]
|
|
422
|
-
skip = 'pyproject.toml,./*env*,./output,./docs/build,./docs/source/.jupyter_cache,./docs/source/*/data,CHANGELOG.md,uv.lock,requirements.*.txt,*.html,*.lock,*.ipynb,./graphify-out,cosign.pub'
|
|
423
|
-
|
|
405
|
+
skip = 'pyproject.toml,./*env*,./specs,./output,./docs/build,./docs/source/.jupyter_cache,./docs/source/*/data,CHANGELOG.md,uv.lock,requirements.*.txt,*.html,*.lock,*.ipynb,./graphify-out,cosign.pub'
|
|
406
|
+
builtin = "clear,rare,en-GB_to_en-US"
|
|
407
|
+
ignore-words-list = ["Hart","FPR", "MOT", "mot", "reencode", "writeable", "coo"]
|
|
424
408
|
|
|
425
409
|
[build-system]
|
|
426
410
|
requires = ["hatchling", "hatch-vcs"]
|
|
@@ -43,8 +43,10 @@ def log(level: int = logging.DEBUG, handler: logging.Handler | None = None) -> N
|
|
|
43
43
|
"""
|
|
44
44
|
Add a handler to the logger quickly for debugging.
|
|
45
45
|
|
|
46
|
-
Calling this more than once is idempotent
|
|
47
|
-
|
|
46
|
+
Calling this more than once is idempotent. With the default handler, a later call
|
|
47
|
+
reuses the handler an earlier call attached. A custom handler equal to one already
|
|
48
|
+
attached to the logger is not added again. Either way, log lines are not
|
|
49
|
+
duplicated, and each call sets the level.
|
|
48
50
|
|
|
49
51
|
Parameters
|
|
50
52
|
----------
|
|
@@ -58,15 +60,21 @@ def log(level: int = logging.DEBUG, handler: logging.Handler | None = None) -> N
|
|
|
58
60
|
"""
|
|
59
61
|
import logging
|
|
60
62
|
|
|
63
|
+
from dataeval._log import ConsoleHandler
|
|
64
|
+
|
|
61
65
|
_logger = logging.getLogger(_ROOT)
|
|
62
66
|
if handler is None:
|
|
63
|
-
handler =
|
|
67
|
+
handler = next((h for h in _logger.handlers if isinstance(h, ConsoleHandler)), None)
|
|
68
|
+
if handler is None:
|
|
69
|
+
handler = ConsoleHandler()
|
|
64
70
|
handler.setFormatter(
|
|
65
71
|
logging.Formatter(
|
|
66
72
|
"%(asctime)s %(levelname)-8s %(name)s.%(filename)s:%(lineno)s - %(funcName)10s() | %(message)s",
|
|
67
73
|
),
|
|
68
74
|
)
|
|
69
|
-
|
|
75
|
+
attached = handler not in _logger.handlers
|
|
76
|
+
if attached:
|
|
70
77
|
_logger.addHandler(handler)
|
|
71
78
|
_logger.setLevel(level)
|
|
72
|
-
|
|
79
|
+
if attached:
|
|
80
|
+
_logger.debug("Added logging handler %s to logger: %s", handler, _ROOT)
|