euler-eval 2.14.0__tar.gz → 2.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_eval-2.14.0 → euler_eval-2.16.0}/PKG-INFO +47 -2
- {euler_eval-2.14.0 → euler_eval-2.16.0}/README.md +46 -1
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/cli.py +243 -5
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/data.py +182 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/evaluate.py +683 -32
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/__init__.py +2 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/points3d_cloud.py +62 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/PKG-INFO +47 -2
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/SOURCES.txt +1 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/pyproject.toml +1 -1
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_data_builders.py +37 -0
- euler_eval-2.16.0/tests/test_points_3d_sparse.py +544 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/__init__.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/config_paths.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/absrel.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/daniel_error.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/depth_binned_error.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/depth_edge_f1.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/depth_standard.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/fid_kid.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/gpu_depth_batch.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/gpu_image_batch.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/high_freq_energy.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/lpips_metric.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/normal_consistency.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/points3d_decomposition.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/points3d_distance.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/points3d_geometry.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/psnr.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rgb_edge_f1.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rgb_lpips.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rgb_psnr_ssim.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rho_a.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rmse.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/scale_invariant_log.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/ssim.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/tail_errors.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/utils.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/sanity_checker.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/utils/hierarchy_parser.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/dependency_links.txt +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/entry_points.txt +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/requires.txt +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/top_level.txt +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/init_cache.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/setup.cfg +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_alignment.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_cli_device.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_config.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_data.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_depth_alignment_output.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_depth_standard.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_evaluate_helpers.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_init_cache.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_integration.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_meta_output.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_metrics_utils.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_points_3d.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_rgb_fid_output.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_rho_a.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_save_results.py +0 -0
- {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_sparse_depth.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: euler-eval
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.16.0
|
|
4
4
|
Summary: Depth map evaluation toolkit with comprehensive metrics
|
|
5
5
|
Author: Depth Eval Contributors
|
|
6
6
|
License: MIT
|
|
@@ -45,7 +45,7 @@ A comprehensive evaluation toolkit for comparing predicted depth maps, RGB image
|
|
|
45
45
|
- **Depth metrics**: PSNR, SSIM, LPIPS, FID, KID, AbsRel, RMSE, Scale-Invariant Log Error, Normal Consistency, Depth Edge F1
|
|
46
46
|
- **RGB metrics**: PSNR, SSIM, LPIPS, FID, SCE (Structural Chromatic Error), Edge F1, Tail Errors (p95/p99), High-Frequency Energy Ratio, Depth-Binned Photometric Error
|
|
47
47
|
- **Rays metrics**: ρ_A (AUC of angular accuracy curve), Angular Error statistics and threshold percentages
|
|
48
|
-
- **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces.
|
|
48
|
+
- **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces. Also evaluates a **dense depth prediction against sparse pointcloud GT** (`gt.sparse_depth`): the depth is unprojected with the GT intrinsics and scored in 3D via directed Chamfer completeness / recall plus per-correspondence 3D error.
|
|
49
49
|
- **Benchmark binning**: Optional depth-range benchmark that subdivides metrics into square-root-scaled near/mid/far bins
|
|
50
50
|
- **Sanity checking**: Automatic validation of metric results against configurable thresholds, with detailed warning reports
|
|
51
51
|
- **Sky masking**: Optional exclusion of sky regions from metrics using GT segmentation
|
|
@@ -174,8 +174,19 @@ depth-eval config.json --rgb-fid-backend clean-fid
|
|
|
174
174
|
depth-eval config.json --benchmark-depth-range 0.01 80.0
|
|
175
175
|
|
|
176
176
|
# Evaluate dense depth predictions against sparse pointcloud GT
|
|
177
|
+
# (emits pointwise sparse-depth metrics AND 3D points_3d metrics)
|
|
177
178
|
depth-eval example_sparse_depth_config.json --skip-rgb --skip-rays
|
|
178
179
|
|
|
180
|
+
# Sparse pointcloud GT, but only the 3D (points_3d) metrics
|
|
181
|
+
depth-eval example_sparse_depth_config.json --skip-depth --skip-rgb --skip-rays
|
|
182
|
+
|
|
183
|
+
# Sparse pointcloud GT, but only the pointwise depth metrics
|
|
184
|
+
depth-eval example_sparse_depth_config.json --skip-points-3d --skip-rgb --skip-rays
|
|
185
|
+
|
|
186
|
+
# Evaluate a predicted points_3d map directly against sparse pointcloud GT
|
|
187
|
+
# (config uses datasets[].points_3d; align relative point maps with a similarity gauge)
|
|
188
|
+
depth-eval config.json --skip-depth --skip-rgb --skip-rays --points-3d-alignment similarity
|
|
189
|
+
|
|
179
190
|
# Skip rays evaluation
|
|
180
191
|
depth-eval config.json --skip-rays
|
|
181
192
|
|
|
@@ -252,6 +263,13 @@ Each modality entry can optionally include a `split` field to select a specific
|
|
|
252
263
|
|
|
253
264
|
For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.depth`. The prediction uses a dense depth-like map under `datasets[].depth`, `datasets[].relative_depth`, or `datasets[].affine_depth`. The evaluator projects the sparse GT point cloud into the prediction image plane using `gt.intrinsics` and `gt.camera_extrinsics`, then computes pointwise depth metrics only at projected valid pixels. For MUSES through `euler_loading.loaders.muses`, `gt.camera_extrinsics` normally resolves to the direct `lidar2rgb` transform and no extra lidar pose is needed. If a dataset exposes separate lidar and camera poses in a shared frame instead, provide optional `gt.lidar_extrinsics`; the evaluator composes `inv(camera_pose) @ lidar_pose` before projection.
|
|
254
265
|
|
|
266
|
+
When the GT is a sparse pointcloud, the evaluator also produces **3D (`points_3d`) metrics** unless `--skip-points-3d` is set (see [Sparse Points-3D Metrics](#sparse-points-3d-metrics)). The prediction scored in 3D can be either:
|
|
267
|
+
|
|
268
|
+
- a **predicted point map** (`datasets[].points_3d`, an `(H,W,3)` map) — evaluated *directly* against the sparse GT cloud, with the 3D **similarity** gauge selected by `--points-3d-alignment`; or
|
|
269
|
+
- a **dense depth map** (`datasets[].depth`/`relative_depth`/`affine_depth`) — unprojected with `gt.intrinsics` into a point map, with the depth **affine** gauge selected by `--depth-alignment`.
|
|
270
|
+
|
|
271
|
+
If a dataset entry provides both, the predicted `points_3d` map is preferred (the depth is still scored by the pointwise sparse-depth metrics). Either way the sparse GT cloud is projected into the prediction plane, and the 3D results are written to a separate `points3d_eval.json` (or `<output_file>_points3d.json`) so they never clobber the sparse-depth `eval.json`.
|
|
272
|
+
|
|
255
273
|
Sparse depth does not require segmentation GT. `gt.segmentation` and
|
|
256
274
|
`gt.semantic_segmentation` are optional aliases for the same sky-mask source.
|
|
257
275
|
They are loaded only when `--mask-sky` is set; in sparse depth evaluation that
|
|
@@ -423,6 +441,31 @@ The angular `error_decomposition` ray metrics and `rho_a` are computed on the
|
|
|
423
441
|
**native** point-map directions (the camera-faithful frame); the radial/lateral and
|
|
424
442
|
Euclidean metrics are reported per space.
|
|
425
443
|
|
|
444
|
+
### Sparse Points-3D Metrics
|
|
445
|
+
|
|
446
|
+
When the ground truth is a sparse pointcloud (`gt.sparse_depth`), the evaluator
|
|
447
|
+
additionally scores the prediction as a 3D point map (unless `--skip-points-3d`
|
|
448
|
+
is set). The sparse GT cloud is projected into the prediction plane, giving both
|
|
449
|
+
the visible GT cloud and per-pixel correspondences. The predicted point map is
|
|
450
|
+
either taken **directly** from `datasets[].points_3d` (gauge = 3D similarity via
|
|
451
|
+
`--points-3d-alignment`) or **unprojected** from a dense `datasets[].depth`
|
|
452
|
+
prediction with `gt.intrinsics` (gauge = depth affine via `--depth-alignment`).
|
|
453
|
+
Results serialize under `points3d.eval.{native,metric}` in a separate output
|
|
454
|
+
file.
|
|
455
|
+
|
|
456
|
+
Only the categories that stay meaningful with a sparse GT are emitted —
|
|
457
|
+
dense-neighbourhood `geometric` metrics (normals, edge F1) are skipped, and the
|
|
458
|
+
`cloud_distance` block reports only the **directed `gt→pred`** side (a correct
|
|
459
|
+
dense prediction has many legitimate points far from any sparse GT return, so
|
|
460
|
+
`accuracy`/`precision` would be misleading and are omitted):
|
|
461
|
+
|
|
462
|
+
| Metric | Key (per space) | Description |
|
|
463
|
+
|---|---|---|
|
|
464
|
+
| Cloud completeness | `cloud_distance.chamfer.completeness`, `.median` | Mean / median distance from each GT return to its nearest predicted point (metres, lower is better) |
|
|
465
|
+
| Cloud recall | `cloud_distance.fscore.tau_<τ>.recall` | Fraction of GT returns within τ m of a predicted point, `τ ∈ {0.05, 0.1, 0.25, 0.5}` (higher is better) |
|
|
466
|
+
| 3D point error | `point_error.{image_mean,image_median,pixel_pool}.*` | 3D EPE `mae3d`/`rmse3d`, percentiles, relative error, and δ-accuracy at the projected correspondences |
|
|
467
|
+
| Error decomposition | `error_decomposition.{radial_*,lateral_*,lateral_fraction,angular_error.*,rho_a.*}` | Radial (≈ depth) vs lateral (≈ camera-model) split at correspondences, with ray angular error and ρ_A |
|
|
468
|
+
|
|
426
469
|
## Output
|
|
427
470
|
|
|
428
471
|
Results are saved as JSON per modality per prediction dataset (one file for depth or sparse depth, one for RGB, one for rays). Default path: `eval.json` inside each modality's dataset path, unless overridden by `output_file` in the config.
|
|
@@ -493,6 +536,8 @@ For sparse depth outputs, the internal Python result dict still uses `sparse_dep
|
|
|
493
536
|
|
|
494
537
|
For points_3d outputs, the internal Python result dict uses `points_3d_native`, `points_3d_metric`, and `points_3d` (canonical alias), but serialized `eval.json` metric paths are rooted at `points3d.eval` (no underscore) to satisfy the namespace first-segment rule, mirroring `sparsedepth`. The `metric` space is emitted only when a gauge alignment is applied; otherwise only `native` is present and aliased.
|
|
495
538
|
|
|
539
|
+
When the points_3d metrics come from a sparse pointcloud GT (`gt.sparse_depth` + a predicted `points_3d` map or a dense depth prediction), they use the same `points3d.eval` namespace but are written to a distinct file — `points3d_eval.json` beside the prediction, or `<output_file>_points3d.json` when `output_file` is set — so they do not overwrite the sparse-depth `eval.json`. Only the `point_error`, `error_decomposition`, and (directed) `cloud_distance` categories are present; the `metric` space is emitted only when the gauge (`--points-3d-alignment` for a point-map prediction, `--depth-alignment` for a depth prediction) calibrates the prediction.
|
|
540
|
+
|
|
496
541
|
Previous single-depth structure (kept under `depth`) is:
|
|
497
542
|
|
|
498
543
|
```json
|
|
@@ -7,7 +7,7 @@ A comprehensive evaluation toolkit for comparing predicted depth maps, RGB image
|
|
|
7
7
|
- **Depth metrics**: PSNR, SSIM, LPIPS, FID, KID, AbsRel, RMSE, Scale-Invariant Log Error, Normal Consistency, Depth Edge F1
|
|
8
8
|
- **RGB metrics**: PSNR, SSIM, LPIPS, FID, SCE (Structural Chromatic Error), Edge F1, Tail Errors (p95/p99), High-Frequency Energy Ratio, Depth-Binned Photometric Error
|
|
9
9
|
- **Rays metrics**: ρ_A (AUC of angular accuracy curve), Angular Error statistics and threshold percentages
|
|
10
|
-
- **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces.
|
|
10
|
+
- **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces. Also evaluates a **dense depth prediction against sparse pointcloud GT** (`gt.sparse_depth`): the depth is unprojected with the GT intrinsics and scored in 3D via directed Chamfer completeness / recall plus per-correspondence 3D error.
|
|
11
11
|
- **Benchmark binning**: Optional depth-range benchmark that subdivides metrics into square-root-scaled near/mid/far bins
|
|
12
12
|
- **Sanity checking**: Automatic validation of metric results against configurable thresholds, with detailed warning reports
|
|
13
13
|
- **Sky masking**: Optional exclusion of sky regions from metrics using GT segmentation
|
|
@@ -136,8 +136,19 @@ depth-eval config.json --rgb-fid-backend clean-fid
|
|
|
136
136
|
depth-eval config.json --benchmark-depth-range 0.01 80.0
|
|
137
137
|
|
|
138
138
|
# Evaluate dense depth predictions against sparse pointcloud GT
|
|
139
|
+
# (emits pointwise sparse-depth metrics AND 3D points_3d metrics)
|
|
139
140
|
depth-eval example_sparse_depth_config.json --skip-rgb --skip-rays
|
|
140
141
|
|
|
142
|
+
# Sparse pointcloud GT, but only the 3D (points_3d) metrics
|
|
143
|
+
depth-eval example_sparse_depth_config.json --skip-depth --skip-rgb --skip-rays
|
|
144
|
+
|
|
145
|
+
# Sparse pointcloud GT, but only the pointwise depth metrics
|
|
146
|
+
depth-eval example_sparse_depth_config.json --skip-points-3d --skip-rgb --skip-rays
|
|
147
|
+
|
|
148
|
+
# Evaluate a predicted points_3d map directly against sparse pointcloud GT
|
|
149
|
+
# (config uses datasets[].points_3d; align relative point maps with a similarity gauge)
|
|
150
|
+
depth-eval config.json --skip-depth --skip-rgb --skip-rays --points-3d-alignment similarity
|
|
151
|
+
|
|
141
152
|
# Skip rays evaluation
|
|
142
153
|
depth-eval config.json --skip-rays
|
|
143
154
|
|
|
@@ -214,6 +225,13 @@ Each modality entry can optionally include a `split` field to select a specific
|
|
|
214
225
|
|
|
215
226
|
For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.depth`. The prediction uses a dense depth-like map under `datasets[].depth`, `datasets[].relative_depth`, or `datasets[].affine_depth`. The evaluator projects the sparse GT point cloud into the prediction image plane using `gt.intrinsics` and `gt.camera_extrinsics`, then computes pointwise depth metrics only at projected valid pixels. For MUSES through `euler_loading.loaders.muses`, `gt.camera_extrinsics` normally resolves to the direct `lidar2rgb` transform and no extra lidar pose is needed. If a dataset exposes separate lidar and camera poses in a shared frame instead, provide optional `gt.lidar_extrinsics`; the evaluator composes `inv(camera_pose) @ lidar_pose` before projection.
|
|
216
227
|
|
|
228
|
+
When the GT is a sparse pointcloud, the evaluator also produces **3D (`points_3d`) metrics** unless `--skip-points-3d` is set (see [Sparse Points-3D Metrics](#sparse-points-3d-metrics)). The prediction scored in 3D can be either:
|
|
229
|
+
|
|
230
|
+
- a **predicted point map** (`datasets[].points_3d`, an `(H,W,3)` map) — evaluated *directly* against the sparse GT cloud, with the 3D **similarity** gauge selected by `--points-3d-alignment`; or
|
|
231
|
+
- a **dense depth map** (`datasets[].depth`/`relative_depth`/`affine_depth`) — unprojected with `gt.intrinsics` into a point map, with the depth **affine** gauge selected by `--depth-alignment`.
|
|
232
|
+
|
|
233
|
+
If a dataset entry provides both, the predicted `points_3d` map is preferred (the depth is still scored by the pointwise sparse-depth metrics). Either way the sparse GT cloud is projected into the prediction plane, and the 3D results are written to a separate `points3d_eval.json` (or `<output_file>_points3d.json`) so they never clobber the sparse-depth `eval.json`.
|
|
234
|
+
|
|
217
235
|
Sparse depth does not require segmentation GT. `gt.segmentation` and
|
|
218
236
|
`gt.semantic_segmentation` are optional aliases for the same sky-mask source.
|
|
219
237
|
They are loaded only when `--mask-sky` is set; in sparse depth evaluation that
|
|
@@ -385,6 +403,31 @@ The angular `error_decomposition` ray metrics and `rho_a` are computed on the
|
|
|
385
403
|
**native** point-map directions (the camera-faithful frame); the radial/lateral and
|
|
386
404
|
Euclidean metrics are reported per space.
|
|
387
405
|
|
|
406
|
+
### Sparse Points-3D Metrics
|
|
407
|
+
|
|
408
|
+
When the ground truth is a sparse pointcloud (`gt.sparse_depth`), the evaluator
|
|
409
|
+
additionally scores the prediction as a 3D point map (unless `--skip-points-3d`
|
|
410
|
+
is set). The sparse GT cloud is projected into the prediction plane, giving both
|
|
411
|
+
the visible GT cloud and per-pixel correspondences. The predicted point map is
|
|
412
|
+
either taken **directly** from `datasets[].points_3d` (gauge = 3D similarity via
|
|
413
|
+
`--points-3d-alignment`) or **unprojected** from a dense `datasets[].depth`
|
|
414
|
+
prediction with `gt.intrinsics` (gauge = depth affine via `--depth-alignment`).
|
|
415
|
+
Results serialize under `points3d.eval.{native,metric}` in a separate output
|
|
416
|
+
file.
|
|
417
|
+
|
|
418
|
+
Only the categories that stay meaningful with a sparse GT are emitted —
|
|
419
|
+
dense-neighbourhood `geometric` metrics (normals, edge F1) are skipped, and the
|
|
420
|
+
`cloud_distance` block reports only the **directed `gt→pred`** side (a correct
|
|
421
|
+
dense prediction has many legitimate points far from any sparse GT return, so
|
|
422
|
+
`accuracy`/`precision` would be misleading and are omitted):
|
|
423
|
+
|
|
424
|
+
| Metric | Key (per space) | Description |
|
|
425
|
+
|---|---|---|
|
|
426
|
+
| Cloud completeness | `cloud_distance.chamfer.completeness`, `.median` | Mean / median distance from each GT return to its nearest predicted point (metres, lower is better) |
|
|
427
|
+
| Cloud recall | `cloud_distance.fscore.tau_<τ>.recall` | Fraction of GT returns within τ m of a predicted point, `τ ∈ {0.05, 0.1, 0.25, 0.5}` (higher is better) |
|
|
428
|
+
| 3D point error | `point_error.{image_mean,image_median,pixel_pool}.*` | 3D EPE `mae3d`/`rmse3d`, percentiles, relative error, and δ-accuracy at the projected correspondences |
|
|
429
|
+
| Error decomposition | `error_decomposition.{radial_*,lateral_*,lateral_fraction,angular_error.*,rho_a.*}` | Radial (≈ depth) vs lateral (≈ camera-model) split at correspondences, with ray angular error and ρ_A |
|
|
430
|
+
|
|
388
431
|
## Output
|
|
389
432
|
|
|
390
433
|
Results are saved as JSON per modality per prediction dataset (one file for depth or sparse depth, one for RGB, one for rays). Default path: `eval.json` inside each modality's dataset path, unless overridden by `output_file` in the config.
|
|
@@ -455,6 +498,8 @@ For sparse depth outputs, the internal Python result dict still uses `sparse_dep
|
|
|
455
498
|
|
|
456
499
|
For points_3d outputs, the internal Python result dict uses `points_3d_native`, `points_3d_metric`, and `points_3d` (canonical alias), but serialized `eval.json` metric paths are rooted at `points3d.eval` (no underscore) to satisfy the namespace first-segment rule, mirroring `sparsedepth`. The `metric` space is emitted only when a gauge alignment is applied; otherwise only `native` is present and aliased.
|
|
457
500
|
|
|
501
|
+
When the points_3d metrics come from a sparse pointcloud GT (`gt.sparse_depth` + a predicted `points_3d` map or a dense depth prediction), they use the same `points3d.eval` namespace but are written to a distinct file — `points3d_eval.json` beside the prediction, or `<output_file>_points3d.json` when `output_file` is set — so they do not overwrite the sparse-depth `eval.json`. Only the `point_error`, `error_decomposition`, and (directed) `cloud_distance` categories are present; the `metric` space is emitted only when the gauge (`--points-3d-alignment` for a point-map prediction, `--depth-alignment` for a depth prediction) calibrates the prediction.
|
|
502
|
+
|
|
458
503
|
Previous single-depth structure (kept under `depth`) is:
|
|
459
504
|
|
|
460
505
|
```json
|
|
@@ -18,6 +18,7 @@ from .config_paths import normalize_modality_path
|
|
|
18
18
|
from .data import (
|
|
19
19
|
build_depth_eval_dataset,
|
|
20
20
|
build_points_3d_eval_dataset,
|
|
21
|
+
build_points_3d_sparse_eval_dataset,
|
|
21
22
|
build_rays_eval_dataset,
|
|
22
23
|
build_rgb_eval_dataset,
|
|
23
24
|
build_sparse_depth_eval_dataset,
|
|
@@ -30,6 +31,7 @@ from .data import (
|
|
|
30
31
|
from .evaluate import (
|
|
31
32
|
evaluate_depth_samples,
|
|
32
33
|
evaluate_points_3d_samples,
|
|
34
|
+
evaluate_points_3d_sparse_samples,
|
|
33
35
|
evaluate_rays_samples,
|
|
34
36
|
evaluate_rgb_samples,
|
|
35
37
|
evaluate_sparse_depth_samples,
|
|
@@ -973,7 +975,12 @@ def _save_json_to_zip(zip_path: Path, internal_name: str, data: dict) -> None:
|
|
|
973
975
|
|
|
974
976
|
|
|
975
977
|
def save_results(
|
|
976
|
-
results: dict,
|
|
978
|
+
results: dict,
|
|
979
|
+
dataset_config: dict,
|
|
980
|
+
modality: str | None = None,
|
|
981
|
+
*,
|
|
982
|
+
default_basename: str = "eval.json",
|
|
983
|
+
output_file_suffix: str | None = None,
|
|
977
984
|
) -> Path:
|
|
978
985
|
"""Save results to output file.
|
|
979
986
|
|
|
@@ -987,6 +994,14 @@ def save_results(
|
|
|
987
994
|
modality: When set, save to this specific modality's path
|
|
988
995
|
(e.g. ``"depth"`` or ``"rgb"``). Falls back to the first
|
|
989
996
|
available modality path when *None*.
|
|
997
|
+
default_basename: File name used when no explicit ``output_file`` is
|
|
998
|
+
configured (default ``"eval.json"``). Lets a second modality that
|
|
999
|
+
shares a prediction path (e.g. sparse ``points_3d`` alongside sparse
|
|
1000
|
+
``depth``) write to a distinct file instead of clobbering it.
|
|
1001
|
+
output_file_suffix: When an explicit ``output_file`` is configured, this
|
|
1002
|
+
suffix is inserted before its extension (e.g. ``"_points3d"`` turns
|
|
1003
|
+
``model_a.json`` into ``model_a_points3d.json``) so multiple
|
|
1004
|
+
modalities can honour a single configured path without collision.
|
|
990
1005
|
|
|
991
1006
|
Returns:
|
|
992
1007
|
Path where results were saved.
|
|
@@ -1006,7 +1021,7 @@ def save_results(
|
|
|
1006
1021
|
dataset_config[selected_modality]["path"],
|
|
1007
1022
|
split=dataset_config[selected_modality].get("split"),
|
|
1008
1023
|
)
|
|
1009
|
-
/
|
|
1024
|
+
/ default_basename
|
|
1010
1025
|
)
|
|
1011
1026
|
else:
|
|
1012
1027
|
# Default: save alongside first available modality path
|
|
@@ -1017,13 +1032,17 @@ def save_results(
|
|
|
1017
1032
|
dataset_config[mod]["path"],
|
|
1018
1033
|
split=dataset_config[mod].get("split"),
|
|
1019
1034
|
)
|
|
1020
|
-
/
|
|
1035
|
+
/ default_basename
|
|
1021
1036
|
)
|
|
1022
1037
|
break
|
|
1023
1038
|
if output_file is None:
|
|
1024
|
-
output_file = Path(
|
|
1039
|
+
output_file = Path(default_basename)
|
|
1025
1040
|
else:
|
|
1026
1041
|
output_file = Path(output_file)
|
|
1042
|
+
if output_file_suffix:
|
|
1043
|
+
output_file = output_file.with_name(
|
|
1044
|
+
f"{output_file.stem}{output_file_suffix}{output_file.suffix}"
|
|
1045
|
+
)
|
|
1027
1046
|
|
|
1028
1047
|
zip_path, internal_name = _find_zip_ancestor(output_file)
|
|
1029
1048
|
if zip_path is not None and internal_name:
|
|
@@ -1273,6 +1292,7 @@ def main():
|
|
|
1273
1292
|
rgb_save = {}
|
|
1274
1293
|
rays_save = {}
|
|
1275
1294
|
points_3d_save = {}
|
|
1295
|
+
points_3d_sparse_save = {}
|
|
1276
1296
|
et_eval_datasets = {}
|
|
1277
1297
|
has_benchmark = args.benchmark_depth_range is not None
|
|
1278
1298
|
|
|
@@ -2103,13 +2123,218 @@ def main():
|
|
|
2103
2123
|
f"POINTS_3D: {ds_name}",
|
|
2104
2124
|
)
|
|
2105
2125
|
|
|
2106
|
-
elif
|
|
2126
|
+
elif (
|
|
2127
|
+
has_points_3d
|
|
2128
|
+
and not args.skip_points_3d
|
|
2129
|
+
and not gt_sparse_depth_path
|
|
2130
|
+
):
|
|
2131
|
+
# (When gt.sparse_depth is present the sparse points_3d branch below
|
|
2132
|
+
# evaluates the predicted point map against the LiDAR cloud instead.)
|
|
2107
2133
|
print(
|
|
2108
2134
|
f"\n[POINTS_3D] Skipping '{ds_name}': no usable GT. Provide "
|
|
2109
2135
|
"gt.points_3d, or gt.depth with gt.intrinsics/gt.calibration to "
|
|
2110
2136
|
"synthesize the GT point map."
|
|
2111
2137
|
)
|
|
2112
2138
|
|
|
2139
|
+
# -- Points-3D against sparse pointcloud GT --
|
|
2140
|
+
# When the GT is a sparse LiDAR cloud (gt.sparse_depth), evaluate the
|
|
2141
|
+
# *3D* metrics (Chamfer/F-score completeness + per-correspondence 3D
|
|
2142
|
+
# error) against the sparse GT cloud. Two prediction sources are
|
|
2143
|
+
# supported, with the native point map preferred when both are present:
|
|
2144
|
+
# - a predicted points_3d map, scored directly (3D similarity gauge);
|
|
2145
|
+
# - a dense depth map, unprojected with the GT intrinsics (affine
|
|
2146
|
+
# gauge). Independent of the sparse-depth pointwise branch; gated by
|
|
2147
|
+
# --skip-points-3d.
|
|
2148
|
+
if (
|
|
2149
|
+
gt_sparse_depth_path
|
|
2150
|
+
and (has_points_3d or has_depth)
|
|
2151
|
+
and not args.skip_points_3d
|
|
2152
|
+
):
|
|
2153
|
+
p3s_pred_is_depth = not has_points_3d
|
|
2154
|
+
print(f"\n[POINTS_3D · SPARSE] Evaluating: '{ds_name}'")
|
|
2155
|
+
print(f" GT sparse pointcloud: {gt_sparse_depth_path}")
|
|
2156
|
+
if p3s_pred_is_depth:
|
|
2157
|
+
p3s_pred_path = pred_depth_config["path"]
|
|
2158
|
+
p3s_pred_split = pred_depth_config.get("split")
|
|
2159
|
+
p3s_pred_entry = pred_depth_key
|
|
2160
|
+
print(f" Pred dense depth: {p3s_pred_path}")
|
|
2161
|
+
if pred_depth_key != "depth":
|
|
2162
|
+
print(f" Pred depth entry: {pred_depth_key}")
|
|
2163
|
+
points_3d_sparse_dataset = build_sparse_depth_eval_dataset(
|
|
2164
|
+
gt_sparse_depth_path=gt_sparse_depth_path,
|
|
2165
|
+
pred_depth_path=p3s_pred_path,
|
|
2166
|
+
intrinsics_path=intrinsics_path,
|
|
2167
|
+
camera_extrinsics_path=camera_extrinsics_path,
|
|
2168
|
+
lidar_extrinsics_path=lidar_extrinsics_path,
|
|
2169
|
+
segmentation_path=segmentation_path,
|
|
2170
|
+
pred_depth_metadata_scope=(
|
|
2171
|
+
pred_depth_key if pred_depth_key != "depth" else None
|
|
2172
|
+
),
|
|
2173
|
+
segmentation_modality_key=segmentation_key or "segmentation",
|
|
2174
|
+
gt_sparse_depth_split=gt_sparse_depth_split,
|
|
2175
|
+
pred_depth_split=p3s_pred_split,
|
|
2176
|
+
intrinsics_split=intrinsics_split,
|
|
2177
|
+
camera_extrinsics_split=camera_extrinsics_split,
|
|
2178
|
+
lidar_extrinsics_split=lidar_extrinsics_split,
|
|
2179
|
+
segmentation_split=segmentation_split,
|
|
2180
|
+
)
|
|
2181
|
+
p3ds_meta = get_sparse_depth_metadata(points_3d_sparse_dataset)
|
|
2182
|
+
p3s_pred_is_radial = p3ds_meta["pred_radial_depth"]
|
|
2183
|
+
p3s_alignment_mode = args.depth_alignment
|
|
2184
|
+
p3s_input_hint = _prediction_depth_space_hint(pred_depth_key)
|
|
2185
|
+
print(f" pred_radial_depth: {p3s_pred_is_radial}")
|
|
2186
|
+
else:
|
|
2187
|
+
p3s_pred_path = dataset_config["points_3d"]["path"]
|
|
2188
|
+
p3s_pred_split = dataset_config["points_3d"].get("split")
|
|
2189
|
+
p3s_pred_entry = "points_3d"
|
|
2190
|
+
print(f" Pred point map: {p3s_pred_path}")
|
|
2191
|
+
points_3d_sparse_dataset = build_points_3d_sparse_eval_dataset(
|
|
2192
|
+
gt_sparse_depth_path=gt_sparse_depth_path,
|
|
2193
|
+
pred_points_3d_path=p3s_pred_path,
|
|
2194
|
+
intrinsics_path=intrinsics_path,
|
|
2195
|
+
camera_extrinsics_path=camera_extrinsics_path,
|
|
2196
|
+
lidar_extrinsics_path=lidar_extrinsics_path,
|
|
2197
|
+
segmentation_path=segmentation_path,
|
|
2198
|
+
segmentation_modality_key=segmentation_key or "segmentation",
|
|
2199
|
+
gt_sparse_depth_split=gt_sparse_depth_split,
|
|
2200
|
+
pred_points_3d_split=p3s_pred_split,
|
|
2201
|
+
intrinsics_split=intrinsics_split,
|
|
2202
|
+
camera_extrinsics_split=camera_extrinsics_split,
|
|
2203
|
+
lidar_extrinsics_split=lidar_extrinsics_split,
|
|
2204
|
+
segmentation_split=segmentation_split,
|
|
2205
|
+
)
|
|
2206
|
+
p3ds_meta = get_points_3d_metadata(points_3d_sparse_dataset)
|
|
2207
|
+
p3s_pred_is_radial = True
|
|
2208
|
+
p3s_alignment_mode = args.points_3d_alignment
|
|
2209
|
+
p3s_input_hint = None
|
|
2210
|
+
et_eval_datasets["points_3d_sparse"] = points_3d_sparse_dataset
|
|
2211
|
+
print(f" Matched pairs: {len(points_3d_sparse_dataset)}")
|
|
2212
|
+
|
|
2213
|
+
points_3d_sparse_results = evaluate_points_3d_sparse_samples(
|
|
2214
|
+
dataset=points_3d_sparse_dataset,
|
|
2215
|
+
pred_is_radial=p3s_pred_is_radial,
|
|
2216
|
+
gt_name=gt.get("name", "GT"),
|
|
2217
|
+
pred_name=ds_name,
|
|
2218
|
+
num_workers=args.num_workers,
|
|
2219
|
+
verbose=args.verbose,
|
|
2220
|
+
sanity_checker=sanity_checker,
|
|
2221
|
+
sky_mask_enabled=args.mask_sky,
|
|
2222
|
+
alignment_mode=p3s_alignment_mode,
|
|
2223
|
+
input_space_hint=p3s_input_hint,
|
|
2224
|
+
pred_is_depth=p3s_pred_is_depth,
|
|
2225
|
+
)
|
|
2226
|
+
|
|
2227
|
+
if sanity_checker is not None:
|
|
2228
|
+
sanity_checker.print_pair_report(ds_name, modality="points_3d")
|
|
2229
|
+
|
|
2230
|
+
p3s_space_info = points_3d_sparse_results.get("space_info", {})
|
|
2231
|
+
p3s_dataset_info = points_3d_sparse_results.get("dataset_info", {})
|
|
2232
|
+
p3s_spatial = points_3d_sparse_results.get("spatial_info", {})
|
|
2233
|
+
|
|
2234
|
+
points_3d_sparse_ns = _EvalNamespace(
|
|
2235
|
+
producer="euler-eval",
|
|
2236
|
+
producer_version=_get_version(),
|
|
2237
|
+
modalities=("points_3d",),
|
|
2238
|
+
axes=_points_3d_eval_axes(),
|
|
2239
|
+
descriptions=_POINTS_3D_EVAL_DESCRIPTIONS,
|
|
2240
|
+
)
|
|
2241
|
+
points_3d_sparse_save = {
|
|
2242
|
+
"metricSet": _points_3d_metric_set_envelope(
|
|
2243
|
+
points_3d_sparse_ns,
|
|
2244
|
+
metadata={
|
|
2245
|
+
"input_space_detected": p3s_space_info.get(
|
|
2246
|
+
"input_space_detected", "unknown"
|
|
2247
|
+
),
|
|
2248
|
+
"metric_space_source": p3s_space_info.get(
|
|
2249
|
+
"metric_space_source"
|
|
2250
|
+
),
|
|
2251
|
+
"calibration_mode": p3s_space_info.get(
|
|
2252
|
+
"calibration_mode", "unknown"
|
|
2253
|
+
),
|
|
2254
|
+
"calibration_applied": p3s_space_info.get(
|
|
2255
|
+
"calibration_applied", False
|
|
2256
|
+
),
|
|
2257
|
+
"emitted_spaces": p3s_space_info.get("emitted_spaces", []),
|
|
2258
|
+
"canonical_space": p3s_space_info.get(
|
|
2259
|
+
"canonical_space", "native"
|
|
2260
|
+
),
|
|
2261
|
+
"gt_representation": "point_cloud",
|
|
2262
|
+
"fov_domain": p3s_dataset_info.get("fov_domain"),
|
|
2263
|
+
"threshold_deg": p3s_dataset_info.get("threshold_deg"),
|
|
2264
|
+
},
|
|
2265
|
+
),
|
|
2266
|
+
"dataset_info": p3s_dataset_info,
|
|
2267
|
+
"meta": _clean_metric_tree({
|
|
2268
|
+
"version": _get_version(),
|
|
2269
|
+
"modality": "points_3d",
|
|
2270
|
+
"device": args.device,
|
|
2271
|
+
"gt": {
|
|
2272
|
+
"path": gt_sparse_depth_path,
|
|
2273
|
+
"split": gt_sparse_depth_split,
|
|
2274
|
+
"representation": "point_cloud",
|
|
2275
|
+
},
|
|
2276
|
+
"pred": {
|
|
2277
|
+
"path": p3s_pred_path,
|
|
2278
|
+
"split": p3s_pred_split,
|
|
2279
|
+
"entry": p3s_pred_entry,
|
|
2280
|
+
"representation": "depth" if p3s_pred_is_depth else "points_3d",
|
|
2281
|
+
"dimensions": p3s_spatial.get("pred_dimensions"),
|
|
2282
|
+
},
|
|
2283
|
+
"calibration": {
|
|
2284
|
+
"intrinsics_path": intrinsics_path,
|
|
2285
|
+
"intrinsics_split": intrinsics_split,
|
|
2286
|
+
"camera_extrinsics_path": camera_extrinsics_path,
|
|
2287
|
+
"camera_extrinsics_split": camera_extrinsics_split,
|
|
2288
|
+
"lidar_extrinsics_path": lidar_extrinsics_path,
|
|
2289
|
+
"lidar_extrinsics_split": lidar_extrinsics_split,
|
|
2290
|
+
},
|
|
2291
|
+
"spatial_alignment": {
|
|
2292
|
+
"method": p3s_spatial.get("method", "pointcloud_projection"),
|
|
2293
|
+
"evaluated_dimensions": p3s_spatial.get(
|
|
2294
|
+
"evaluated_dimensions"
|
|
2295
|
+
),
|
|
2296
|
+
},
|
|
2297
|
+
"modality_params": p3ds_meta,
|
|
2298
|
+
"eval_params": {
|
|
2299
|
+
"sky_masking": args.mask_sky,
|
|
2300
|
+
"alignment_mode": p3s_alignment_mode,
|
|
2301
|
+
"num_workers": args.num_workers,
|
|
2302
|
+
},
|
|
2303
|
+
}),
|
|
2304
|
+
"points3d": {"eval": {}},
|
|
2305
|
+
}
|
|
2306
|
+
for space_name, result_key in (
|
|
2307
|
+
("native", "points_3d_native"),
|
|
2308
|
+
("metric", "points_3d_metric"),
|
|
2309
|
+
):
|
|
2310
|
+
branch = points_3d_sparse_results.get(result_key)
|
|
2311
|
+
if branch is not None:
|
|
2312
|
+
points_3d_sparse_save["points3d"]["eval"][space_name] = (
|
|
2313
|
+
_clean_metric_tree(branch)
|
|
2314
|
+
)
|
|
2315
|
+
p3s_pfm = points_3d_sparse_results.get("per_file_metrics", {})
|
|
2316
|
+
if p3s_pfm:
|
|
2317
|
+
canonical_space = p3s_space_info.get("canonical_space", "native")
|
|
2318
|
+
points_3d_sparse_save["per_file_metrics"] = _clean_per_file_metrics(
|
|
2319
|
+
_wrap_pfm_metrics(
|
|
2320
|
+
p3s_pfm,
|
|
2321
|
+
lambda m: _wrap_depth_space_pfm_metrics(
|
|
2322
|
+
m,
|
|
2323
|
+
metric_root="points3d",
|
|
2324
|
+
canonical_key="points_3d",
|
|
2325
|
+
canonical_space=canonical_space,
|
|
2326
|
+
),
|
|
2327
|
+
),
|
|
2328
|
+
label="points_3d sparse per_file_metrics",
|
|
2329
|
+
)
|
|
2330
|
+
all_results.setdefault("per_file_metrics", {}).update(p3s_pfm)
|
|
2331
|
+
|
|
2332
|
+
print_results(
|
|
2333
|
+
{k: v for k, v in points_3d_sparse_results.items()
|
|
2334
|
+
if k not in ("per_file_metrics", "spatial_info")},
|
|
2335
|
+
f"POINTS_3D · SPARSE: {ds_name}",
|
|
2336
|
+
)
|
|
2337
|
+
|
|
2113
2338
|
# Save per-modality results to respective dataset paths
|
|
2114
2339
|
if depth_save:
|
|
2115
2340
|
depth_out = save_results(
|
|
@@ -2134,6 +2359,19 @@ def main():
|
|
|
2134
2359
|
points_3d_save, dataset_config, modality="points_3d"
|
|
2135
2360
|
)
|
|
2136
2361
|
print(f"\n Points-3D results saved to: {points_3d_out}")
|
|
2362
|
+
if points_3d_sparse_save:
|
|
2363
|
+
# Distinct file so it never clobbers the sparse-depth eval.json that
|
|
2364
|
+
# shares the same dense-depth prediction path / output_file.
|
|
2365
|
+
points_3d_sparse_out = save_results(
|
|
2366
|
+
points_3d_sparse_save,
|
|
2367
|
+
dataset_config,
|
|
2368
|
+
modality="points_3d",
|
|
2369
|
+
default_basename="points3d_eval.json",
|
|
2370
|
+
output_file_suffix="_points3d",
|
|
2371
|
+
)
|
|
2372
|
+
print(
|
|
2373
|
+
f"\n Points-3D (sparse GT) results saved to: {points_3d_sparse_out}"
|
|
2374
|
+
)
|
|
2137
2375
|
|
|
2138
2376
|
# Log to euler_train
|
|
2139
2377
|
if et_run is not None and et_eval_datasets:
|