euler-eval 2.30.0__tar.gz → 2.31.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_eval-2.30.0/euler_eval.egg-info → euler_eval-2.31.0}/PKG-INFO +4 -3
- {euler_eval-2.30.0 → euler_eval-2.31.0}/README.md +2 -1
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/alignment.md +2 -1
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/cli.md +16 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/configuration.md +77 -14
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/metrics.md +82 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/output.md +59 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/cli.py +430 -101
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/config_paths.py +78 -2
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/data.py +72 -46
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/evaluate.py +359 -11
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/__init__.py +18 -0
- euler_eval-2.31.0/euler_eval/metrics/sky_iou.py +167 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0/euler_eval.egg-info}/PKG-INFO +4 -3
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/SOURCES.txt +5 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/requires.txt +1 -1
- {euler_eval-2.30.0 → euler_eval-2.31.0}/pyproject.toml +2 -2
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/conftest.py +2 -2
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_config.py +148 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_data_builders.py +10 -5
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_evaluate_helpers.py +20 -6
- euler_eval-2.31.0/tests/test_mask_sky_pairing.py +197 -0
- euler_eval-2.31.0/tests/test_mask_sky_prediction.py +239 -0
- euler_eval-2.31.0/tests/test_sky_iou.py +550 -0
- euler_eval-2.31.0/tests/test_sky_iou_cli.py +497 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/LICENSE +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/MANIFEST.in +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/README.md +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/distributions.md +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/feature-request-unified-evaluation-framework.md +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/validation.md +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/THIRD_PARTY_NOTICES.md +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/__init__.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/calibration.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/distribution_contract.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/distributions.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metric_sets.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/_natural_scene_models.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/absrel.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/daniel_error.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/depth_binned_error.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/depth_edge_f1.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/depth_standard.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/fade.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/fid_kid.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/gpu_depth_batch.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/gpu_image_batch.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/high_freq_energy.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/lpips_metric.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/niqe.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/normal_consistency.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/points3d_cloud.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/points3d_decomposition.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/points3d_distance.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/points3d_geometry.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/psnr.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rgb_edge_f1.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rgb_lpips.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rgb_psnr_ssim.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rho_a.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rmse.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/scale_invariant_log.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/ssim.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/tail_errors.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/utils.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/sanity_checker.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/schemas/distribution-v1.schema.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/utils/hierarchy_parser.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/validation.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/dependency_links.txt +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/entry_points.txt +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/top_level.txt +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/init_cache.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/scripts/generate_distribution_fixtures.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/setup.cfg +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/__init__.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/distribution_contract_utils.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/README.md +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/counts-depth.eval.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/counts-points3d.eval.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/counts-sparsedepth.eval.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/future-sum-by-depth.eval.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/histogram-only.eval.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/per-file-only.eval.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/scalar-only.eval.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/mock_files/calibration_output.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/mock_files/depth_output.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/mock_files/rgb_output.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/mock_files/segmentation_output.json +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_alignment.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_cli_device.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_cli_sparse_outputs.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_data.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_depth_alignment_output.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_depth_standard.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_distribution_contract.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_distributions.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_domain_metrics.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_init_cache.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_integration.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_meta_output.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_metrics_utils.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_normal_consistency.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_points_3d.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_points_3d_sparse.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_rgb_fid_output.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_rho_a.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_save_results.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_sparse_depth.py +0 -0
- {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_validation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: euler-eval
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.31.0
|
|
4
4
|
Summary: Evaluation toolkit for depth, RGB, ray and 3D point-map predictions
|
|
5
5
|
Author-email: Daniel Rothenpieler <rothenpielerdaniel@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -33,7 +33,7 @@ Requires-Dist: opencv-python-headless>=4.5.0
|
|
|
33
33
|
Requires-Dist: lpips>=0.1.4
|
|
34
34
|
Requires-Dist: torchmetrics>=1.0
|
|
35
35
|
Requires-Dist: tqdm>=4.62.0
|
|
36
|
-
Requires-Dist: euler-loading>=2.
|
|
36
|
+
Requires-Dist: euler-loading>=2.29
|
|
37
37
|
Requires-Dist: ds-crawler>=2.10
|
|
38
38
|
Requires-Dist: euler-metric-naming>=0.2
|
|
39
39
|
Provides-Extra: logging
|
|
@@ -176,7 +176,8 @@ metric is written under, is in [Metrics](docs/metrics.md).
|
|
|
176
176
|
| **Metadata-driven decoding** | Radial vs planar depth, value ranges, point-cloud column layout and loader choice all come from the dataset index — no per-dataset flags. |
|
|
177
177
|
| **Honest spaces** | Relative models are scored `native` *and* `metric` (after scale/shift or a similarity gauge), so a scale failure never reads as a geometry failure. |
|
|
178
178
|
| **Benchmark depth bins** | `--benchmark-depth-range MIN MAX` adds near/mid/far bins in square-root depth space, additive to the regular metrics. |
|
|
179
|
-
| **Sky masking** | `--mask-sky` drops sky pixels using GT segmentation — from the metrics *and* from the alignment fit. |
|
|
179
|
+
| **Sky masking** | `--mask-sky` drops sky pixels using GT segmentation — from the metrics *and* from the alignment fit. `--mask-sky-prediction` does the same with each prediction's own `sky_mask`. |
|
|
180
|
+
| **Sky IoU** | `--sky-iou` scores predicted sky — a `sky_mask`, or depth reaching a fixed cap — against GT segmentation, without changing any depth metric. |
|
|
180
181
|
| **Per-file + aggregate** | Dataset-level numbers plus per-image metrics in dataset-hierarchy order, in the same file. |
|
|
181
182
|
| **Sanity checks** | Results are validated against configurable thresholds; implausible ranges, degenerate inputs and scale mismatches are reported instead of quietly scored. |
|
|
182
183
|
| **Structured metric names** | Each result carries a `metricSet` envelope declaring its namespace, axes, units and metric directions. |
|
|
@@ -126,7 +126,8 @@ metric is written under, is in [Metrics](docs/metrics.md).
|
|
|
126
126
|
| **Metadata-driven decoding** | Radial vs planar depth, value ranges, point-cloud column layout and loader choice all come from the dataset index — no per-dataset flags. |
|
|
127
127
|
| **Honest spaces** | Relative models are scored `native` *and* `metric` (after scale/shift or a similarity gauge), so a scale failure never reads as a geometry failure. |
|
|
128
128
|
| **Benchmark depth bins** | `--benchmark-depth-range MIN MAX` adds near/mid/far bins in square-root depth space, additive to the regular metrics. |
|
|
129
|
-
| **Sky masking** | `--mask-sky` drops sky pixels using GT segmentation — from the metrics *and* from the alignment fit. |
|
|
129
|
+
| **Sky masking** | `--mask-sky` drops sky pixels using GT segmentation — from the metrics *and* from the alignment fit. `--mask-sky-prediction` does the same with each prediction's own `sky_mask`. |
|
|
130
|
+
| **Sky IoU** | `--sky-iou` scores predicted sky — a `sky_mask`, or depth reaching a fixed cap — against GT segmentation, without changing any depth metric. |
|
|
130
131
|
| **Per-file + aggregate** | Dataset-level numbers plus per-image metrics in dataset-hierarchy order, in the same file. |
|
|
131
132
|
| **Sanity checks** | Results are validated against configurable thresholds; implausible ranges, degenerate inputs and scale mismatches are reported instead of quietly scored. |
|
|
132
133
|
| **Structured metric names** | Each result carries a `metricSet` envelope declaring its namespace, axes, units and metric directions. |
|
|
@@ -32,7 +32,8 @@ map `d_aligned = s · d_pred + t` by least squares over valid pixels.
|
|
|
32
32
|
|
|
33
33
|
The fit uses pixels where GT and prediction are finite and GT > 0. With
|
|
34
34
|
`--mask-sky`, sky pixels are excluded and the fit is trimmed to `gt ≤ P95(gt)`
|
|
35
|
-
so residual sky outliers do not dominate.
|
|
35
|
+
so residual sky outliers do not dominate. Pixels in a prediction's `sky_mask`
|
|
36
|
+
are excluded from the fit with `--sky-depth` or `--mask-sky-prediction`.
|
|
36
37
|
|
|
37
38
|
Declaring the prediction with the `relative_depth` or `affine_depth` config key
|
|
38
39
|
(instead of `depth`) tells the evaluator the model is not metric, so
|
|
@@ -23,7 +23,9 @@ validation exits with status `1` before any dataset is opened.
|
|
|
23
23
|
| `--skip-rays` | flag | off | Skip rays (spherical direction map) evaluation |
|
|
24
24
|
| `--skip-points-3d` | flag | off | Skip points_3d (per-pixel 3D point map) evaluation |
|
|
25
25
|
| `--mask-sky` | flag | off | Mask sky regions from metrics using GT segmentation |
|
|
26
|
+
| `--mask-sky-prediction` | flag | off | Exclude pixels in a prediction's `datasets[].sky_mask` from dense and sparse depth metrics and the alignment fit; predictions without a `sky_mask` are scored in full ([details](configuration.md#masking-predicted-sky)) |
|
|
26
27
|
| `--sky-depth` | `float` | none | Cap GT and predicted depths in meters and fill predicted `sky_mask` pixels with this value; must be finite and positive |
|
|
28
|
+
| `--sky-iou` | flag | off | Score predicted sky against GT segmentation sky: IoU, precision and recall for the `explicit` (`datasets[].sky_mask`) and `implicit` (`depth >= --sky-depth`, metric predictions only) variants. Requires `gt.segmentation`; depth metrics are unchanged ([details](metrics.md#sky-segmentation)) |
|
|
27
29
|
| `--no-sanity-check` | flag | off | Disable sanity checking of metric configurations |
|
|
28
30
|
| `--metrics-config` | `str` | auto-detect | Path to `metrics_config.json` for sanity checking |
|
|
29
31
|
| `--depth-alignment` | `{none,auto_affine,affine}` | `auto_affine` | Depth calibration mode ([details](alignment.md#depth-alignment)) |
|
|
@@ -54,9 +56,16 @@ euler-eval config.json --batch-size 32
|
|
|
54
56
|
# Sky masking (requires gt.segmentation)
|
|
55
57
|
euler-eval config.json --mask-sky -v
|
|
56
58
|
|
|
59
|
+
# Exclude each prediction's own sky (datasets[].sky_mask) from depth metrics
|
|
60
|
+
euler-eval config.json --mask-sky-prediction
|
|
61
|
+
|
|
57
62
|
# Cap depth at 100 m and fill predicted sky at 100 m (optional datasets[].sky_mask)
|
|
58
63
|
euler-eval config.json --sky-depth 100
|
|
59
64
|
|
|
65
|
+
# Sky IoU: explicit from datasets[].sky_mask, implicit at depth >= 100 m
|
|
66
|
+
# (requires gt.segmentation; the threshold is also the depth cap)
|
|
67
|
+
euler-eval config.json --sky-iou --sky-depth 100
|
|
68
|
+
|
|
60
69
|
# Only depth evaluation
|
|
61
70
|
euler-eval config.json --skip-rgb --skip-rays --skip-points-3d
|
|
62
71
|
|
|
@@ -109,6 +118,13 @@ per prediction dataset and modality with the GT and prediction paths, the
|
|
|
109
118
|
resolved modality metadata (such as `radial_depth`), and the number of matched
|
|
110
119
|
pairs.
|
|
111
120
|
|
|
121
|
+
With `--sky-iou`, each depth block ends with one line holding the pooled
|
|
122
|
+
(`pixel_pool`) sky IoU, precision and recall of every computed variant, for
|
|
123
|
+
example `Sky IoU, pixel_pool, 'model': explicit 0.9120 (precision 0.9500,
|
|
124
|
+
recall 0.9580); implicit (depth >= 100 m) 0.4410 (precision 0.9800, recall
|
|
125
|
+
0.4450)`. Skipped variants, GT segmentation without sky and an empty implicit
|
|
126
|
+
sky are reported as warnings above it.
|
|
127
|
+
|
|
112
128
|
A matched-pair count that is much lower than expected is the first sign that GT
|
|
113
129
|
and predictions are not indexed consistently — see
|
|
114
130
|
[Loader resolution & dataset metadata](configuration.md#loader-resolution--dataset-metadata).
|
|
@@ -71,7 +71,8 @@ required.
|
|
|
71
71
|
| `gt.sparse_depth.path` | no\* | Sparse pointcloud GT, `(N,C)` points whose first columns are `x,y,z` in metres |
|
|
72
72
|
| `gt.rays.path` | no\* | GT ray direction map dataset |
|
|
73
73
|
| `gt.points_3d.path` | no\* | GT per-pixel 3D point map `(H,W,3)` in camera-frame metres |
|
|
74
|
-
| `gt.segmentation.path` | no | GT segmentation (needed for `--mask-sky`) |
|
|
74
|
+
| `gt.segmentation.path` | no | GT segmentation (needed for `--mask-sky` and `--sky-iou`); see [GT segmentation](#gt-segmentation) |
|
|
75
|
+
| `gt.semantic_segmentation.path` | no | Alias of `gt.segmentation`; configure only one of the two |
|
|
75
76
|
| `gt.calibration.path` | no | Calibration data (camera intrinsics); enables true camera-frame [surface normals](metrics.md#surface-normals-and-calibration) |
|
|
76
77
|
| `gt.intrinsics.path` | with `gt.sparse_depth` | Camera intrinsics for pointcloud projection |
|
|
77
78
|
| `gt.camera_extrinsics.path` | with `gt.sparse_depth` | Source-to-camera extrinsics (e.g. MUSES `lidar2rgb`) |
|
|
@@ -80,6 +81,35 @@ required.
|
|
|
80
81
|
|
|
81
82
|
\* At least one of the five modality entries must be present.
|
|
82
83
|
|
|
84
|
+
### GT segmentation
|
|
85
|
+
|
|
86
|
+
`gt.segmentation` is the canonical key. `gt.semantic_segmentation` is accepted
|
|
87
|
+
as an alias, for archives whose segmentation scope carries that name (MUSES,
|
|
88
|
+
for example); configuring both keys is an error. The segmentation is read only
|
|
89
|
+
when `--mask-sky` or `--sky-iou` needs it. `--mask-sky` without it warns and
|
|
90
|
+
disables itself, while `--sky-iou` without it exits with an error.
|
|
91
|
+
|
|
92
|
+
The dataset head's loader module must expose a `sky_mask` function, and the
|
|
93
|
+
head must declare the sky label as `skyclass` (euler-loading 2.29 or later).
|
|
94
|
+
euler-eval contains no label IDs.
|
|
95
|
+
|
|
96
|
+
Segmentation is per frame, so it is paired with the evaluated frames by
|
|
97
|
+
hierarchy path and file ID, like depth: its index must use the same folder
|
|
98
|
+
structure and IDs as the frames it masks. A frame without its own segmentation
|
|
99
|
+
file is not evaluated, and the run prints how many pairs the segmentation
|
|
100
|
+
dropped. When segmentation is read only for `--sky-iou`, it must not change
|
|
101
|
+
the evaluated pairs, so a segmentation that lacks frames is not loaded; the run
|
|
102
|
+
warns and the sky IoU is skipped. A segmentation whose IDs match no frame
|
|
103
|
+
stops the run with euler-loading's "No common IDs" error and per-modality file
|
|
104
|
+
counts.
|
|
105
|
+
|
|
106
|
+
The config key is also the default metadata scope. If the path has no
|
|
107
|
+
`#scope=` selector, no root-level metadata, and no scope of that name, the
|
|
108
|
+
scope is chosen among the archive's scopes: the one whose name or
|
|
109
|
+
`dataset-head.json` modality key is `segmentation` or `semantic_segmentation`
|
|
110
|
+
(both names are candidates under either key), or the only scope. If several
|
|
111
|
+
scopes match, or none among several, the run stops and asks for `#scope=`.
|
|
112
|
+
|
|
83
113
|
## `datasets` section
|
|
84
114
|
|
|
85
115
|
Each entry needs a `name` and at least one prediction modality. Use only **one**
|
|
@@ -94,7 +124,7 @@ would be ambiguous.
|
|
|
94
124
|
| `depth.path` | no\* | Predicted dense metric depth; also used against sparse pointcloud GT |
|
|
95
125
|
| `relative_depth.path` | no\* | Predicted dense relative depth (scale/shift alignment supported) |
|
|
96
126
|
| `affine_depth.path` | no\* | Predicted dense affine depth (scale/shift alignment supported) |
|
|
97
|
-
| `sky_mask.path` | no | Auxiliary predicted sky mask for a dense depth entry; used with `--sky-depth` |
|
|
127
|
+
| `sky_mask.path` | no | Auxiliary predicted sky mask for a dense depth entry; used with `--sky-depth`, `--mask-sky-prediction` and `--sky-iou` |
|
|
98
128
|
| `rays.path` | no\* | Predicted ray direction map dataset |
|
|
99
129
|
| `points_3d.path` | no\* | Predicted per-pixel 3D point map `(H,W,3)` |
|
|
100
130
|
| `output_file` | no | Custom results path (default: `eval.json` in the first modality path) |
|
|
@@ -123,8 +153,8 @@ The sky mask is paired with GT and predicted depth by sample ID and hierarchy.
|
|
|
123
153
|
It supports the same archive, split, and `#scope=` selectors as depth, defaulting
|
|
124
154
|
to the `sky_mask` metadata scope. Its loader must return a boolean mask with
|
|
125
155
|
**True meaning sky**. For example, `generic_dense_depth.sky_mask` uses the RGB
|
|
126
|
-
sky color in the mask dataset's `meta.
|
|
127
|
-
`(1, H, W)` tensor. Masks at another resolution follow the depth grid's crop
|
|
156
|
+
sky color in the mask dataset's `meta.skyclass` (`[R, G, B]`; the older
|
|
157
|
+
`meta.sky_mask` key is deprecated) and returns a `(1, H, W)` tensor. Masks at another resolution follow the depth grid's crop
|
|
128
158
|
rule or use nearest-neighbor resizing.
|
|
129
159
|
|
|
130
160
|
`--sky-depth` applies to dense and sparse depth evaluation. GT and both native
|
|
@@ -139,9 +169,37 @@ contributes only at projected points. Predicted sky stays in the scores where
|
|
|
139
169
|
GT is valid; `--mask-sky` separately excludes pixels using GT segmentation and
|
|
140
170
|
can be combined with `--sky-depth`.
|
|
141
171
|
|
|
142
|
-
Without `--sky-depth`,
|
|
143
|
-
|
|
144
|
-
|
|
172
|
+
Without `--sky-depth`, `--mask-sky-prediction` or `--sky-iou`, the predicted
|
|
173
|
+
mask is not loaded and depth evaluation uses the usual uncapped values.
|
|
174
|
+
`--sky-iou` loads the mask
|
|
175
|
+
even without `--sky-depth`, but only to score it against GT sky
|
|
176
|
+
([Sky segmentation](metrics.md#sky-segmentation)); the depth fill and the
|
|
177
|
+
affine-fit exclusion stay tied to `--sky-depth` (and `--mask-sky-prediction`),
|
|
178
|
+
so depth values do not change.
|
|
179
|
+
Because the mask is paired by sample ID, a mask that lacks some depth frames
|
|
180
|
+
also drops them from the depth metrics; the run prints how many. When the mask
|
|
181
|
+
is loaded only for `--sky-iou`, such a mask is not loaded at all: the run
|
|
182
|
+
warns, keeps every depth pair and skips the explicit variant. The effective cap and mask
|
|
183
|
+
source are recorded in `eval.json` under `meta.eval_params.sky_depth` and
|
|
184
|
+
`meta.pred.sky_mask`.
|
|
185
|
+
|
|
186
|
+
### Masking predicted sky
|
|
187
|
+
|
|
188
|
+
`--mask-sky-prediction` excludes the pixels of a prediction's `sky_mask` from
|
|
189
|
+
dense and sparse depth evaluation, as `--mask-sky` does for GT sky: from every
|
|
190
|
+
metric, benchmark bin and histogram, and from the scale/shift fit. For sparse
|
|
191
|
+
depth, lidar points that project into predicted sky are not scored. A dataset
|
|
192
|
+
without a `sky_mask` is scored in full and the run warns. The flag loads the
|
|
193
|
+
mask without `--sky-depth`, combines with `--mask-sky` (the union of both skies
|
|
194
|
+
is excluded) and with `--sky-depth` (the cap still applies to the remaining
|
|
195
|
+
pixels). It does not change `--sky-iou`, which scores the masks themselves.
|
|
196
|
+
|
|
197
|
+
The excluded pixels come from the evaluated model, so two models are scored on
|
|
198
|
+
different pixel sets, and a model can lower its error by calling hard pixels
|
|
199
|
+
sky. Each result records how much it excluded under `meta.pred_sky_masking`
|
|
200
|
+
([Results & output](output.md#predicted-sky-masking)); compare models with
|
|
201
|
+
this flag only alongside that fraction, or prefer `--mask-sky`, whose GT sky is
|
|
202
|
+
the same for every model.
|
|
145
203
|
|
|
146
204
|
## Sparse depth (pointcloud GT)
|
|
147
205
|
|
|
@@ -162,8 +220,9 @@ depth and `points_3d`, each prediction modality gets its own `eval.json` at its
|
|
|
162
220
|
own path.
|
|
163
221
|
|
|
164
222
|
Sparse depth does not require segmentation GT. `gt.segmentation` is loaded only
|
|
165
|
-
when `--mask-sky` is set
|
|
166
|
-
metrics and from scale/shift fitting
|
|
223
|
+
when `--mask-sky` or `--sky-iou` is set. `--mask-sky` excludes sky pixels from
|
|
224
|
+
projected-point metrics and from scale/shift fitting; `--sky-iou` scores the
|
|
225
|
+
dense prediction's sky against it without changing any depth metric.
|
|
167
226
|
|
|
168
227
|
```json
|
|
169
228
|
{
|
|
@@ -239,11 +298,15 @@ The scope defaults to the configuration key name, so the common case is compact:
|
|
|
239
298
|
|
|
240
299
|
Use `#scope=<scope>` when an artifact scope is not named exactly like the config
|
|
241
300
|
key, e.g. `"sparse_depth": { "path": "/data/capture.zip:test#scope=lidar" }`.
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
`
|
|
246
|
-
|
|
301
|
+
GT segmentation is the exception: under either of its keys it also finds a
|
|
302
|
+
scope named after the other key, as described in [GT segmentation](#gt-segmentation).
|
|
303
|
+
|
|
304
|
+
Regular modalities (`rgb`, `depth`, `rays`, `points_3d`, `sparse_depth`, GT
|
|
305
|
+
segmentation and a predicted `sky_mask`) participate in file-ID intersection;
|
|
306
|
+
calibration and pose data (`intrinsics`, `calibration`, `camera_extrinsics`,
|
|
307
|
+
`lidar_extrinsics`) are loaded hierarchically and do not. A hierarchical file
|
|
308
|
+
applies to every frame at or below its folder, which suits one calibration per
|
|
309
|
+
scene but not per-frame data.
|
|
247
310
|
|
|
248
311
|
## `metrics_config.json`
|
|
249
312
|
|
|
@@ -76,6 +76,87 @@ the 45° sanity threshold.
|
|
|
76
76
|
> plane read as nearly fronto-parallel and the focal length was unused. Values
|
|
77
77
|
> are not comparable with earlier releases.
|
|
78
78
|
|
|
79
|
+
### Sky segmentation
|
|
80
|
+
|
|
81
|
+
`--sky-iou` scores how well each prediction's sky matches the GT segmentation
|
|
82
|
+
sky. It is opt-in and read-only: no depth value, mask or alignment fit changes,
|
|
83
|
+
so every other key keeps its value. It requires `gt.segmentation` (or its alias
|
|
84
|
+
`gt.semantic_segmentation`); the sky label comes from that dataset head's
|
|
85
|
+
`skyclass`, as for `--mask-sky`.
|
|
86
|
+
|
|
87
|
+
There are two variants. They answer different questions, so they are reported
|
|
88
|
+
under separate keys and never merged:
|
|
89
|
+
|
|
90
|
+
| Variant | Predicted sky `P` | Computed when |
|
|
91
|
+
|---|---|---|
|
|
92
|
+
| `explicit` | The prediction's `datasets[].sky_mask` (True = sky) | The prediction has a `sky_mask` |
|
|
93
|
+
| `implicit` | `depth >= --sky-depth` | `--sky-depth` is set and the prediction is metric |
|
|
94
|
+
|
|
95
|
+
The implicit test reads the prediction after planar-to-radial conversion and
|
|
96
|
+
before calibration and capping. `+inf` counts as sky; NaN and values `<= 0` do
|
|
97
|
+
not. It is not computed for `relative_depth`, `affine_depth` or predictions
|
|
98
|
+
detected as normalized, since their values are not in meters. When both sources
|
|
99
|
+
exist, both variants are reported side by side; neither replaces the other. A
|
|
100
|
+
dataset with neither prints a warning and writes no sky keys.
|
|
101
|
+
|
|
102
|
+
| Metric | Key |
|
|
103
|
+
|---|---|
|
|
104
|
+
| Sky IoU / precision / recall | `depth.eval.sky_segmentation.{explicit,implicit}.{pixel_pool,image_mean,image_median}.{iou,precision,recall}` |
|
|
105
|
+
| Sparse-depth runs | `sparsedepth.eval.sky_segmentation.{explicit,implicit}.{pixel_pool,image_mean,image_median}.{iou,precision,recall}` |
|
|
106
|
+
| Per file | `{depth,sparsedepth}.eval.sky_segmentation.{explicit,implicit}.{iou,precision,recall}` |
|
|
107
|
+
|
|
108
|
+
`sky_segmentation` sits beside `native` and `metric`, not under them, because
|
|
109
|
+
the IoU does not depend on the evaluated depth space. Sparse-depth runs score
|
|
110
|
+
the dense prediction grid against the GT segmentation exactly like dense runs;
|
|
111
|
+
the sparse GT depth plays no part. Provenance, including the threshold and the
|
|
112
|
+
GT and predicted sky fractions, is recorded under
|
|
113
|
+
[`meta.sky_segmentation`](output.md#sky-segmentation-provenance).
|
|
114
|
+
|
|
115
|
+
If the GT segmentation lacks frames and `--mask-sky` is off, it is not loaded
|
|
116
|
+
and no sky IoU is computed, so that the depth pairs stay the same; see
|
|
117
|
+
[GT segmentation](configuration.md#gt-segmentation).
|
|
118
|
+
|
|
119
|
+
**Definitions.** GT sky `G` is the segmentation of the sample's own frame,
|
|
120
|
+
resized to the prediction grid with the same rule as GT depth (crop for the
|
|
121
|
+
VAE multiple-of-8 pattern, nearest neighbour otherwise). An explicit mask is
|
|
122
|
+
resized the same way. The domain `D` is the full prediction grid. Per sample:
|
|
123
|
+
|
|
124
|
+
```text
|
|
125
|
+
I = |P ∩ G ∩ D| U = |(P ∪ G) ∩ D| T_p = |P ∩ D| T_g = |G ∩ D|
|
|
126
|
+
iou = I / U precision = I / T_p recall = I / T_g
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Each value is undefined when its denominator is 0. `image_mean` and
|
|
130
|
+
`image_median` reduce the defined per-image values. `pixel_pool` is
|
|
131
|
+
`ΣI / ΣU`, `ΣI / ΣT_p` and `ΣI / ΣT_g` over all samples, so a sample with an
|
|
132
|
+
undefined IoU (no sky in `P` or `G`) is left out of the image reducers but
|
|
133
|
+
still adds its zero counts to the pooled sums. Undefined dataset values are
|
|
134
|
+
omitted from `eval.json`.
|
|
135
|
+
|
|
136
|
+
**Interpreting the numbers.**
|
|
137
|
+
|
|
138
|
+
- **Explicit IoU** measures agreement between a dedicated sky prediction and GT
|
|
139
|
+
sky. It does not depend on depth quality.
|
|
140
|
+
- **Implicit IoU** measures whether the depth output reaches `--sky-depth` on
|
|
141
|
+
GT sky pixels. It does not measure whether the regressor recognizes sky. A
|
|
142
|
+
regressor that outputs about 95 m over the sky, as UniDepth V2 does in fog,
|
|
143
|
+
never reaches a 250 m cap; its implicit IoU is close to zero by construction,
|
|
144
|
+
even if it places the sky correctly relative to everything else. The same
|
|
145
|
+
model at `--sky-depth 80` can score high. Implicit IoU therefore depends on
|
|
146
|
+
both the model's saturation behaviour and the cap.
|
|
147
|
+
|
|
148
|
+
Consequences:
|
|
149
|
+
|
|
150
|
+
- Never present implicit and explicit IoU as the same quantity, and always
|
|
151
|
+
state the threshold with an implicit value (`meta.sky_segmentation.threshold_m`).
|
|
152
|
+
- Do not choose `--sky-depth` on the benchmark to maximize implicit IoU. That
|
|
153
|
+
is the same leakage as choosing the fill constant on the test set. Fix the
|
|
154
|
+
threshold beforehand, or report several runs at fixed thresholds.
|
|
155
|
+
- Pooled IoU weights images by their sky area, so a few large-sky images can
|
|
156
|
+
dominate it. `image_median` shows typical behaviour.
|
|
157
|
+
- Precision and recall separate the two failure modes. Low recall with high
|
|
158
|
+
precision usually means the regressor does not saturate at the cap.
|
|
159
|
+
|
|
79
160
|
## Sparse depth
|
|
80
161
|
|
|
81
162
|
With a sparse pointcloud GT, only metrics that stay meaningful at isolated
|
|
@@ -86,6 +167,7 @@ skipped. Results serialize under the `sparsedepth.eval` root:
|
|
|
86
167
|
|---|---|
|
|
87
168
|
| Standard | `sparsedepth.eval.{native,metric}.standard.{image_mean,image_median,pixel_pool}.*` |
|
|
88
169
|
| AbsRel / RMSE / SILog | `sparsedepth.eval.{native,metric}.depth_metrics.*` |
|
|
170
|
+
| Sky segmentation (`--sky-iou`) | `sparsedepth.eval.sky_segmentation.*` ([details](#sky-segmentation)) |
|
|
89
171
|
|
|
90
172
|
## RGB
|
|
91
173
|
|
|
@@ -78,6 +78,65 @@ GT/prediction resolution mismatch was resolved — `none` when they already matc
|
|
|
78
78
|
`vae_crop` for the multiple-of-8 crop that latent models produce, `resize`
|
|
79
79
|
otherwise, and `pointcloud_projection` when GT was a sparse cloud.
|
|
80
80
|
|
|
81
|
+
### Sky segmentation provenance
|
|
82
|
+
|
|
83
|
+
With `--sky-iou`, dense and sparse depth results record
|
|
84
|
+
`meta.eval_params.sky_iou: true` and a `meta.sky_segmentation` block. Without
|
|
85
|
+
the flag, neither field exists. Values below are illustrative:
|
|
86
|
+
|
|
87
|
+
```json
|
|
88
|
+
"sky_segmentation": {
|
|
89
|
+
"explicit": {
|
|
90
|
+
"available": true, "source": "pred.sky_mask", "samples": 500,
|
|
91
|
+
"samples_undefined_iou": 3, "pred_sky_fraction": 0.19
|
|
92
|
+
},
|
|
93
|
+
"implicit": { "available": false, "reason": "prediction is not metric (relative)" },
|
|
94
|
+
"gt_source": "gt.segmentation",
|
|
95
|
+
"domain": "prediction_grid",
|
|
96
|
+
"samples": 500,
|
|
97
|
+
"samples_without_gt": 0,
|
|
98
|
+
"gt_sky_fraction": 0.21
|
|
99
|
+
}
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
| Field | Meaning |
|
|
103
|
+
|---|---|
|
|
104
|
+
| `explicit`, `implicit` | Whether each variant was computed. A computed variant gives its `source`, how many `samples` it scored, how many had an undefined IoU (no sky in prediction or GT), and its pooled predicted sky fraction; the implicit one adds `rule: "depth >= threshold_m"`. A skipped variant gives the `reason` instead |
|
|
105
|
+
| `gt_source` | The config key the GT segmentation came from |
|
|
106
|
+
| `threshold_m` | The `--sky-depth` threshold of the implicit variant; present only when that variant was computed |
|
|
107
|
+
| `domain` | Pixels scored: `prediction_grid`, the full prediction grid |
|
|
108
|
+
| `samples` | Depth pairs evaluated |
|
|
109
|
+
| `samples_without_gt` | Pairs left out because no GT segmentation of their own frame was found |
|
|
110
|
+
| `gt_sky_fraction` | Pooled GT sky fraction over the scored samples |
|
|
111
|
+
|
|
112
|
+
### Predicted-sky masking
|
|
113
|
+
|
|
114
|
+
With `--mask-sky-prediction`, dense and sparse depth results record
|
|
115
|
+
`meta.eval_params.mask_sky_prediction: true` and how much the prediction's own
|
|
116
|
+
sky removed, since that differs from model to model:
|
|
117
|
+
|
|
118
|
+
```json
|
|
119
|
+
"pred_sky_masking": {
|
|
120
|
+
"source": "pred.sky_mask",
|
|
121
|
+
"samples": 500,
|
|
122
|
+
"samples_with_sky_mask": 500,
|
|
123
|
+
"gt_valid_pixels": 1048576000,
|
|
124
|
+
"excluded_gt_pixels": 220200960,
|
|
125
|
+
"excluded_gt_fraction": 0.21
|
|
126
|
+
}
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
`gt_valid_pixels` counts the GT pixels (projected lidar points for sparse
|
|
130
|
+
depth) that would be scored without the flag, after `--mask-sky` if set;
|
|
131
|
+
`excluded_gt_pixels` is how many of them fell in predicted sky.
|
|
132
|
+
|
|
133
|
+
When sky keys are written, the metric-set axes also declare them:
|
|
134
|
+
`sky_segmentation` joins the `category` values, an optional `variant` axis
|
|
135
|
+
(`explicit`, `implicit`) follows `category`, and `space` becomes optional
|
|
136
|
+
because these keys have no depth space. `metricDescriptions` gains `iou`,
|
|
137
|
+
`precision` and `recall` (higher is better, bounds `[0, 1]`). Every other key
|
|
138
|
+
decomposes exactly as before.
|
|
139
|
+
|
|
81
140
|
## Spaces and canonical aliases
|
|
82
141
|
|
|
83
142
|
Depth-like and points-3d results are nested by space:
|