euler-eval 2.14.0__tar.gz → 2.16.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {euler_eval-2.14.0 → euler_eval-2.16.0}/PKG-INFO +47 -2
  2. {euler_eval-2.14.0 → euler_eval-2.16.0}/README.md +46 -1
  3. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/cli.py +243 -5
  4. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/data.py +182 -0
  5. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/evaluate.py +683 -32
  6. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/__init__.py +2 -0
  7. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/points3d_cloud.py +62 -0
  8. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/PKG-INFO +47 -2
  9. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/SOURCES.txt +1 -0
  10. {euler_eval-2.14.0 → euler_eval-2.16.0}/pyproject.toml +1 -1
  11. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_data_builders.py +37 -0
  12. euler_eval-2.16.0/tests/test_points_3d_sparse.py +544 -0
  13. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/__init__.py +0 -0
  14. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/config_paths.py +0 -0
  15. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/absrel.py +0 -0
  16. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/daniel_error.py +0 -0
  17. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/depth_binned_error.py +0 -0
  18. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/depth_edge_f1.py +0 -0
  19. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/depth_standard.py +0 -0
  20. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/fid_kid.py +0 -0
  21. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/gpu_depth_batch.py +0 -0
  22. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/gpu_image_batch.py +0 -0
  23. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/high_freq_energy.py +0 -0
  24. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/lpips_metric.py +0 -0
  25. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/normal_consistency.py +0 -0
  26. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/points3d_decomposition.py +0 -0
  27. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/points3d_distance.py +0 -0
  28. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/points3d_geometry.py +0 -0
  29. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/psnr.py +0 -0
  30. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rgb_edge_f1.py +0 -0
  31. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rgb_lpips.py +0 -0
  32. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rgb_psnr_ssim.py +0 -0
  33. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rho_a.py +0 -0
  34. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/rmse.py +0 -0
  35. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/scale_invariant_log.py +0 -0
  36. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/ssim.py +0 -0
  37. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/tail_errors.py +0 -0
  38. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/metrics/utils.py +0 -0
  39. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/sanity_checker.py +0 -0
  40. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval/utils/hierarchy_parser.py +0 -0
  41. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/dependency_links.txt +0 -0
  42. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/entry_points.txt +0 -0
  43. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/requires.txt +0 -0
  44. {euler_eval-2.14.0 → euler_eval-2.16.0}/euler_eval.egg-info/top_level.txt +0 -0
  45. {euler_eval-2.14.0 → euler_eval-2.16.0}/init_cache.py +0 -0
  46. {euler_eval-2.14.0 → euler_eval-2.16.0}/setup.cfg +0 -0
  47. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_alignment.py +0 -0
  48. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_cli_device.py +0 -0
  49. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_config.py +0 -0
  50. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_data.py +0 -0
  51. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_depth_alignment_output.py +0 -0
  52. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_depth_standard.py +0 -0
  53. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_evaluate_helpers.py +0 -0
  54. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_init_cache.py +0 -0
  55. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_integration.py +0 -0
  56. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_meta_output.py +0 -0
  57. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_metrics_utils.py +0 -0
  58. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_points_3d.py +0 -0
  59. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_rgb_fid_output.py +0 -0
  60. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_rho_a.py +0 -0
  61. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_save_results.py +0 -0
  62. {euler_eval-2.14.0 → euler_eval-2.16.0}/tests/test_sparse_depth.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: euler-eval
3
- Version: 2.14.0
3
+ Version: 2.16.0
4
4
  Summary: Depth map evaluation toolkit with comprehensive metrics
5
5
  Author: Depth Eval Contributors
6
6
  License: MIT
@@ -45,7 +45,7 @@ A comprehensive evaluation toolkit for comparing predicted depth maps, RGB image
45
45
  - **Depth metrics**: PSNR, SSIM, LPIPS, FID, KID, AbsRel, RMSE, Scale-Invariant Log Error, Normal Consistency, Depth Edge F1
46
46
  - **RGB metrics**: PSNR, SSIM, LPIPS, FID, SCE (Structural Chromatic Error), Edge F1, Tail Errors (p95/p99), High-Frequency Energy Ratio, Depth-Binned Photometric Error
47
47
  - **Rays metrics**: ρ_A (AUC of angular accuracy curve), Angular Error statistics and threshold percentages
48
- - **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces.
48
+ - **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces. Also evaluates a **dense depth prediction against sparse pointcloud GT** (`gt.sparse_depth`): the depth is unprojected with the GT intrinsics and scored in 3D via directed Chamfer completeness / recall plus per-correspondence 3D error.
49
49
  - **Benchmark binning**: Optional depth-range benchmark that subdivides metrics into square-root-scaled near/mid/far bins
50
50
  - **Sanity checking**: Automatic validation of metric results against configurable thresholds, with detailed warning reports
51
51
  - **Sky masking**: Optional exclusion of sky regions from metrics using GT segmentation
@@ -174,8 +174,19 @@ depth-eval config.json --rgb-fid-backend clean-fid
174
174
  depth-eval config.json --benchmark-depth-range 0.01 80.0
175
175
 
176
176
  # Evaluate dense depth predictions against sparse pointcloud GT
177
+ # (emits pointwise sparse-depth metrics AND 3D points_3d metrics)
177
178
  depth-eval example_sparse_depth_config.json --skip-rgb --skip-rays
178
179
 
180
+ # Sparse pointcloud GT, but only the 3D (points_3d) metrics
181
+ depth-eval example_sparse_depth_config.json --skip-depth --skip-rgb --skip-rays
182
+
183
+ # Sparse pointcloud GT, but only the pointwise depth metrics
184
+ depth-eval example_sparse_depth_config.json --skip-points-3d --skip-rgb --skip-rays
185
+
186
+ # Evaluate a predicted points_3d map directly against sparse pointcloud GT
187
+ # (config uses datasets[].points_3d; align relative point maps with a similarity gauge)
188
+ depth-eval config.json --skip-depth --skip-rgb --skip-rays --points-3d-alignment similarity
189
+
179
190
  # Skip rays evaluation
180
191
  depth-eval config.json --skip-rays
181
192
 
@@ -252,6 +263,13 @@ Each modality entry can optionally include a `split` field to select a specific
252
263
 
253
264
  For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.depth`. The prediction uses a dense depth-like map under `datasets[].depth`, `datasets[].relative_depth`, or `datasets[].affine_depth`. The evaluator projects the sparse GT point cloud into the prediction image plane using `gt.intrinsics` and `gt.camera_extrinsics`, then computes pointwise depth metrics only at projected valid pixels. For MUSES through `euler_loading.loaders.muses`, `gt.camera_extrinsics` normally resolves to the direct `lidar2rgb` transform and no extra lidar pose is needed. If a dataset exposes separate lidar and camera poses in a shared frame instead, provide optional `gt.lidar_extrinsics`; the evaluator composes `inv(camera_pose) @ lidar_pose` before projection.
254
265
 
266
+ When the GT is a sparse pointcloud, the evaluator also produces **3D (`points_3d`) metrics** unless `--skip-points-3d` is set (see [Sparse Points-3D Metrics](#sparse-points-3d-metrics)). The prediction scored in 3D can be either:
267
+
268
+ - a **predicted point map** (`datasets[].points_3d`, an `(H,W,3)` map) — evaluated *directly* against the sparse GT cloud, with the 3D **similarity** gauge selected by `--points-3d-alignment`; or
269
+ - a **dense depth map** (`datasets[].depth`/`relative_depth`/`affine_depth`) — unprojected with `gt.intrinsics` into a point map, with the depth **affine** gauge selected by `--depth-alignment`.
270
+
271
+ If a dataset entry provides both, the predicted `points_3d` map is preferred (the depth is still scored by the pointwise sparse-depth metrics). Either way the sparse GT cloud is projected into the prediction plane, and the 3D results are written to a separate `points3d_eval.json` (or `<output_file>_points3d.json`) so they never clobber the sparse-depth `eval.json`.
272
+
255
273
  Sparse depth does not require segmentation GT. `gt.segmentation` and
256
274
  `gt.semantic_segmentation` are optional aliases for the same sky-mask source.
257
275
  They are loaded only when `--mask-sky` is set; in sparse depth evaluation that
@@ -423,6 +441,31 @@ The angular `error_decomposition` ray metrics and `rho_a` are computed on the
423
441
  **native** point-map directions (the camera-faithful frame); the radial/lateral and
424
442
  Euclidean metrics are reported per space.
425
443
 
444
+ ### Sparse Points-3D Metrics
445
+
446
+ When the ground truth is a sparse pointcloud (`gt.sparse_depth`), the evaluator
447
+ additionally scores the prediction as a 3D point map (unless `--skip-points-3d`
448
+ is set). The sparse GT cloud is projected into the prediction plane, giving both
449
+ the visible GT cloud and per-pixel correspondences. The predicted point map is
450
+ either taken **directly** from `datasets[].points_3d` (gauge = 3D similarity via
451
+ `--points-3d-alignment`) or **unprojected** from a dense `datasets[].depth`
452
+ prediction with `gt.intrinsics` (gauge = depth affine via `--depth-alignment`).
453
+ Results serialize under `points3d.eval.{native,metric}` in a separate output
454
+ file.
455
+
456
+ Only the categories that stay meaningful with a sparse GT are emitted —
457
+ dense-neighbourhood `geometric` metrics (normals, edge F1) are skipped, and the
458
+ `cloud_distance` block reports only the **directed `gt→pred`** side (a correct
459
+ dense prediction has many legitimate points far from any sparse GT return, so
460
+ `accuracy`/`precision` would be misleading and are omitted):
461
+
462
+ | Metric | Key (per space) | Description |
463
+ |---|---|---|
464
+ | Cloud completeness | `cloud_distance.chamfer.completeness`, `.median` | Mean / median distance from each GT return to its nearest predicted point (metres, lower is better) |
465
+ | Cloud recall | `cloud_distance.fscore.tau_<τ>.recall` | Fraction of GT returns within τ m of a predicted point, `τ ∈ {0.05, 0.1, 0.25, 0.5}` (higher is better) |
466
+ | 3D point error | `point_error.{image_mean,image_median,pixel_pool}.*` | 3D EPE `mae3d`/`rmse3d`, percentiles, relative error, and δ-accuracy at the projected correspondences |
467
+ | Error decomposition | `error_decomposition.{radial_*,lateral_*,lateral_fraction,angular_error.*,rho_a.*}` | Radial (≈ depth) vs lateral (≈ camera-model) split at correspondences, with ray angular error and ρ_A |
468
+
426
469
  ## Output
427
470
 
428
471
  Results are saved as JSON per modality per prediction dataset (one file for depth or sparse depth, one for RGB, one for rays). Default path: `eval.json` inside each modality's dataset path, unless overridden by `output_file` in the config.
@@ -493,6 +536,8 @@ For sparse depth outputs, the internal Python result dict still uses `sparse_dep
493
536
 
494
537
  For points_3d outputs, the internal Python result dict uses `points_3d_native`, `points_3d_metric`, and `points_3d` (canonical alias), but serialized `eval.json` metric paths are rooted at `points3d.eval` (no underscore) to satisfy the namespace first-segment rule, mirroring `sparsedepth`. The `metric` space is emitted only when a gauge alignment is applied; otherwise only `native` is present and aliased.
495
538
 
539
+ When the points_3d metrics come from a sparse pointcloud GT (`gt.sparse_depth` + a predicted `points_3d` map or a dense depth prediction), they use the same `points3d.eval` namespace but are written to a distinct file — `points3d_eval.json` beside the prediction, or `<output_file>_points3d.json` when `output_file` is set — so they do not overwrite the sparse-depth `eval.json`. Only the `point_error`, `error_decomposition`, and (directed) `cloud_distance` categories are present; the `metric` space is emitted only when the gauge (`--points-3d-alignment` for a point-map prediction, `--depth-alignment` for a depth prediction) calibrates the prediction.
540
+
496
541
  Previous single-depth structure (kept under `depth`) is:
497
542
 
498
543
  ```json
@@ -7,7 +7,7 @@ A comprehensive evaluation toolkit for comparing predicted depth maps, RGB image
7
7
  - **Depth metrics**: PSNR, SSIM, LPIPS, FID, KID, AbsRel, RMSE, Scale-Invariant Log Error, Normal Consistency, Depth Edge F1
8
8
  - **RGB metrics**: PSNR, SSIM, LPIPS, FID, SCE (Structural Chromatic Error), Edge F1, Tail Errors (p95/p99), High-Frequency Energy Ratio, Depth-Binned Photometric Error
9
9
  - **Rays metrics**: ρ_A (AUC of angular accuracy curve), Angular Error statistics and threshold percentages
10
- - **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces.
10
+ - **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces. Also evaluates a **dense depth prediction against sparse pointcloud GT** (`gt.sparse_depth`): the depth is unprojected with the GT intrinsics and scored in 3D via directed Chamfer completeness / recall plus per-correspondence 3D error.
11
11
  - **Benchmark binning**: Optional depth-range benchmark that subdivides metrics into square-root-scaled near/mid/far bins
12
12
  - **Sanity checking**: Automatic validation of metric results against configurable thresholds, with detailed warning reports
13
13
  - **Sky masking**: Optional exclusion of sky regions from metrics using GT segmentation
@@ -136,8 +136,19 @@ depth-eval config.json --rgb-fid-backend clean-fid
136
136
  depth-eval config.json --benchmark-depth-range 0.01 80.0
137
137
 
138
138
  # Evaluate dense depth predictions against sparse pointcloud GT
139
+ # (emits pointwise sparse-depth metrics AND 3D points_3d metrics)
139
140
  depth-eval example_sparse_depth_config.json --skip-rgb --skip-rays
140
141
 
142
+ # Sparse pointcloud GT, but only the 3D (points_3d) metrics
143
+ depth-eval example_sparse_depth_config.json --skip-depth --skip-rgb --skip-rays
144
+
145
+ # Sparse pointcloud GT, but only the pointwise depth metrics
146
+ depth-eval example_sparse_depth_config.json --skip-points-3d --skip-rgb --skip-rays
147
+
148
+ # Evaluate a predicted points_3d map directly against sparse pointcloud GT
149
+ # (config uses datasets[].points_3d; align relative point maps with a similarity gauge)
150
+ depth-eval config.json --skip-depth --skip-rgb --skip-rays --points-3d-alignment similarity
151
+
141
152
  # Skip rays evaluation
142
153
  depth-eval config.json --skip-rays
143
154
 
@@ -214,6 +225,13 @@ Each modality entry can optionally include a `split` field to select a specific
214
225
 
215
226
  For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.depth`. The prediction uses a dense depth-like map under `datasets[].depth`, `datasets[].relative_depth`, or `datasets[].affine_depth`. The evaluator projects the sparse GT point cloud into the prediction image plane using `gt.intrinsics` and `gt.camera_extrinsics`, then computes pointwise depth metrics only at projected valid pixels. For MUSES through `euler_loading.loaders.muses`, `gt.camera_extrinsics` normally resolves to the direct `lidar2rgb` transform and no extra lidar pose is needed. If a dataset exposes separate lidar and camera poses in a shared frame instead, provide optional `gt.lidar_extrinsics`; the evaluator composes `inv(camera_pose) @ lidar_pose` before projection.
216
227
 
228
+ When the GT is a sparse pointcloud, the evaluator also produces **3D (`points_3d`) metrics** unless `--skip-points-3d` is set (see [Sparse Points-3D Metrics](#sparse-points-3d-metrics)). The prediction scored in 3D can be either:
229
+
230
+ - a **predicted point map** (`datasets[].points_3d`, an `(H,W,3)` map) — evaluated *directly* against the sparse GT cloud, with the 3D **similarity** gauge selected by `--points-3d-alignment`; or
231
+ - a **dense depth map** (`datasets[].depth`/`relative_depth`/`affine_depth`) — unprojected with `gt.intrinsics` into a point map, with the depth **affine** gauge selected by `--depth-alignment`.
232
+
233
+ If a dataset entry provides both, the predicted `points_3d` map is preferred (the depth is still scored by the pointwise sparse-depth metrics). Either way the sparse GT cloud is projected into the prediction plane, and the 3D results are written to a separate `points3d_eval.json` (or `<output_file>_points3d.json`) so they never clobber the sparse-depth `eval.json`.
234
+
217
235
  Sparse depth does not require segmentation GT. `gt.segmentation` and
218
236
  `gt.semantic_segmentation` are optional aliases for the same sky-mask source.
219
237
  They are loaded only when `--mask-sky` is set; in sparse depth evaluation that
@@ -385,6 +403,31 @@ The angular `error_decomposition` ray metrics and `rho_a` are computed on the
385
403
  **native** point-map directions (the camera-faithful frame); the radial/lateral and
386
404
  Euclidean metrics are reported per space.
387
405
 
406
+ ### Sparse Points-3D Metrics
407
+
408
+ When the ground truth is a sparse pointcloud (`gt.sparse_depth`), the evaluator
409
+ additionally scores the prediction as a 3D point map (unless `--skip-points-3d`
410
+ is set). The sparse GT cloud is projected into the prediction plane, giving both
411
+ the visible GT cloud and per-pixel correspondences. The predicted point map is
412
+ either taken **directly** from `datasets[].points_3d` (gauge = 3D similarity via
413
+ `--points-3d-alignment`) or **unprojected** from a dense `datasets[].depth`
414
+ prediction with `gt.intrinsics` (gauge = depth affine via `--depth-alignment`).
415
+ Results serialize under `points3d.eval.{native,metric}` in a separate output
416
+ file.
417
+
418
+ Only the categories that stay meaningful with a sparse GT are emitted —
419
+ dense-neighbourhood `geometric` metrics (normals, edge F1) are skipped, and the
420
+ `cloud_distance` block reports only the **directed `gt→pred`** side (a correct
421
+ dense prediction has many legitimate points far from any sparse GT return, so
422
+ `accuracy`/`precision` would be misleading and are omitted):
423
+
424
+ | Metric | Key (per space) | Description |
425
+ |---|---|---|
426
+ | Cloud completeness | `cloud_distance.chamfer.completeness`, `.median` | Mean / median distance from each GT return to its nearest predicted point (metres, lower is better) |
427
+ | Cloud recall | `cloud_distance.fscore.tau_<τ>.recall` | Fraction of GT returns within τ m of a predicted point, `τ ∈ {0.05, 0.1, 0.25, 0.5}` (higher is better) |
428
+ | 3D point error | `point_error.{image_mean,image_median,pixel_pool}.*` | 3D EPE `mae3d`/`rmse3d`, percentiles, relative error, and δ-accuracy at the projected correspondences |
429
+ | Error decomposition | `error_decomposition.{radial_*,lateral_*,lateral_fraction,angular_error.*,rho_a.*}` | Radial (≈ depth) vs lateral (≈ camera-model) split at correspondences, with ray angular error and ρ_A |
430
+
388
431
  ## Output
389
432
 
390
433
  Results are saved as JSON per modality per prediction dataset (one file for depth or sparse depth, one for RGB, one for rays). Default path: `eval.json` inside each modality's dataset path, unless overridden by `output_file` in the config.
@@ -455,6 +498,8 @@ For sparse depth outputs, the internal Python result dict still uses `sparse_dep
455
498
 
456
499
  For points_3d outputs, the internal Python result dict uses `points_3d_native`, `points_3d_metric`, and `points_3d` (canonical alias), but serialized `eval.json` metric paths are rooted at `points3d.eval` (no underscore) to satisfy the namespace first-segment rule, mirroring `sparsedepth`. The `metric` space is emitted only when a gauge alignment is applied; otherwise only `native` is present and aliased.
457
500
 
501
+ When the points_3d metrics come from a sparse pointcloud GT (`gt.sparse_depth` + a predicted `points_3d` map or a dense depth prediction), they use the same `points3d.eval` namespace but are written to a distinct file — `points3d_eval.json` beside the prediction, or `<output_file>_points3d.json` when `output_file` is set — so they do not overwrite the sparse-depth `eval.json`. Only the `point_error`, `error_decomposition`, and (directed) `cloud_distance` categories are present; the `metric` space is emitted only when the gauge (`--points-3d-alignment` for a point-map prediction, `--depth-alignment` for a depth prediction) calibrates the prediction.
502
+
458
503
  Previous single-depth structure (kept under `depth`) is:
459
504
 
460
505
  ```json
@@ -18,6 +18,7 @@ from .config_paths import normalize_modality_path
18
18
  from .data import (
19
19
  build_depth_eval_dataset,
20
20
  build_points_3d_eval_dataset,
21
+ build_points_3d_sparse_eval_dataset,
21
22
  build_rays_eval_dataset,
22
23
  build_rgb_eval_dataset,
23
24
  build_sparse_depth_eval_dataset,
@@ -30,6 +31,7 @@ from .data import (
30
31
  from .evaluate import (
31
32
  evaluate_depth_samples,
32
33
  evaluate_points_3d_samples,
34
+ evaluate_points_3d_sparse_samples,
33
35
  evaluate_rays_samples,
34
36
  evaluate_rgb_samples,
35
37
  evaluate_sparse_depth_samples,
@@ -973,7 +975,12 @@ def _save_json_to_zip(zip_path: Path, internal_name: str, data: dict) -> None:
973
975
 
974
976
 
975
977
  def save_results(
976
- results: dict, dataset_config: dict, modality: str | None = None
978
+ results: dict,
979
+ dataset_config: dict,
980
+ modality: str | None = None,
981
+ *,
982
+ default_basename: str = "eval.json",
983
+ output_file_suffix: str | None = None,
977
984
  ) -> Path:
978
985
  """Save results to output file.
979
986
 
@@ -987,6 +994,14 @@ def save_results(
987
994
  modality: When set, save to this specific modality's path
988
995
  (e.g. ``"depth"`` or ``"rgb"``). Falls back to the first
989
996
  available modality path when *None*.
997
+ default_basename: File name used when no explicit ``output_file`` is
998
+ configured (default ``"eval.json"``). Lets a second modality that
999
+ shares a prediction path (e.g. sparse ``points_3d`` alongside sparse
1000
+ ``depth``) write to a distinct file instead of clobbering it.
1001
+ output_file_suffix: When an explicit ``output_file`` is configured, this
1002
+ suffix is inserted before its extension (e.g. ``"_points3d"`` turns
1003
+ ``model_a.json`` into ``model_a_points3d.json``) so multiple
1004
+ modalities can honour a single configured path without collision.
990
1005
 
991
1006
  Returns:
992
1007
  Path where results were saved.
@@ -1006,7 +1021,7 @@ def save_results(
1006
1021
  dataset_config[selected_modality]["path"],
1007
1022
  split=dataset_config[selected_modality].get("split"),
1008
1023
  )
1009
- / "eval.json"
1024
+ / default_basename
1010
1025
  )
1011
1026
  else:
1012
1027
  # Default: save alongside first available modality path
@@ -1017,13 +1032,17 @@ def save_results(
1017
1032
  dataset_config[mod]["path"],
1018
1033
  split=dataset_config[mod].get("split"),
1019
1034
  )
1020
- / "eval.json"
1035
+ / default_basename
1021
1036
  )
1022
1037
  break
1023
1038
  if output_file is None:
1024
- output_file = Path("eval.json")
1039
+ output_file = Path(default_basename)
1025
1040
  else:
1026
1041
  output_file = Path(output_file)
1042
+ if output_file_suffix:
1043
+ output_file = output_file.with_name(
1044
+ f"{output_file.stem}{output_file_suffix}{output_file.suffix}"
1045
+ )
1027
1046
 
1028
1047
  zip_path, internal_name = _find_zip_ancestor(output_file)
1029
1048
  if zip_path is not None and internal_name:
@@ -1273,6 +1292,7 @@ def main():
1273
1292
  rgb_save = {}
1274
1293
  rays_save = {}
1275
1294
  points_3d_save = {}
1295
+ points_3d_sparse_save = {}
1276
1296
  et_eval_datasets = {}
1277
1297
  has_benchmark = args.benchmark_depth_range is not None
1278
1298
 
@@ -2103,13 +2123,218 @@ def main():
2103
2123
  f"POINTS_3D: {ds_name}",
2104
2124
  )
2105
2125
 
2106
- elif has_points_3d and not args.skip_points_3d:
2126
+ elif (
2127
+ has_points_3d
2128
+ and not args.skip_points_3d
2129
+ and not gt_sparse_depth_path
2130
+ ):
2131
+ # (When gt.sparse_depth is present the sparse points_3d branch below
2132
+ # evaluates the predicted point map against the LiDAR cloud instead.)
2107
2133
  print(
2108
2134
  f"\n[POINTS_3D] Skipping '{ds_name}': no usable GT. Provide "
2109
2135
  "gt.points_3d, or gt.depth with gt.intrinsics/gt.calibration to "
2110
2136
  "synthesize the GT point map."
2111
2137
  )
2112
2138
 
2139
+ # -- Points-3D against sparse pointcloud GT --
2140
+ # When the GT is a sparse LiDAR cloud (gt.sparse_depth), evaluate the
2141
+ # *3D* metrics (Chamfer/F-score completeness + per-correspondence 3D
2142
+ # error) against the sparse GT cloud. Two prediction sources are
2143
+ # supported, with the native point map preferred when both are present:
2144
+ # - a predicted points_3d map, scored directly (3D similarity gauge);
2145
+ # - a dense depth map, unprojected with the GT intrinsics (affine
2146
+ # gauge). Independent of the sparse-depth pointwise branch; gated by
2147
+ # --skip-points-3d.
2148
+ if (
2149
+ gt_sparse_depth_path
2150
+ and (has_points_3d or has_depth)
2151
+ and not args.skip_points_3d
2152
+ ):
2153
+ p3s_pred_is_depth = not has_points_3d
2154
+ print(f"\n[POINTS_3D · SPARSE] Evaluating: '{ds_name}'")
2155
+ print(f" GT sparse pointcloud: {gt_sparse_depth_path}")
2156
+ if p3s_pred_is_depth:
2157
+ p3s_pred_path = pred_depth_config["path"]
2158
+ p3s_pred_split = pred_depth_config.get("split")
2159
+ p3s_pred_entry = pred_depth_key
2160
+ print(f" Pred dense depth: {p3s_pred_path}")
2161
+ if pred_depth_key != "depth":
2162
+ print(f" Pred depth entry: {pred_depth_key}")
2163
+ points_3d_sparse_dataset = build_sparse_depth_eval_dataset(
2164
+ gt_sparse_depth_path=gt_sparse_depth_path,
2165
+ pred_depth_path=p3s_pred_path,
2166
+ intrinsics_path=intrinsics_path,
2167
+ camera_extrinsics_path=camera_extrinsics_path,
2168
+ lidar_extrinsics_path=lidar_extrinsics_path,
2169
+ segmentation_path=segmentation_path,
2170
+ pred_depth_metadata_scope=(
2171
+ pred_depth_key if pred_depth_key != "depth" else None
2172
+ ),
2173
+ segmentation_modality_key=segmentation_key or "segmentation",
2174
+ gt_sparse_depth_split=gt_sparse_depth_split,
2175
+ pred_depth_split=p3s_pred_split,
2176
+ intrinsics_split=intrinsics_split,
2177
+ camera_extrinsics_split=camera_extrinsics_split,
2178
+ lidar_extrinsics_split=lidar_extrinsics_split,
2179
+ segmentation_split=segmentation_split,
2180
+ )
2181
+ p3ds_meta = get_sparse_depth_metadata(points_3d_sparse_dataset)
2182
+ p3s_pred_is_radial = p3ds_meta["pred_radial_depth"]
2183
+ p3s_alignment_mode = args.depth_alignment
2184
+ p3s_input_hint = _prediction_depth_space_hint(pred_depth_key)
2185
+ print(f" pred_radial_depth: {p3s_pred_is_radial}")
2186
+ else:
2187
+ p3s_pred_path = dataset_config["points_3d"]["path"]
2188
+ p3s_pred_split = dataset_config["points_3d"].get("split")
2189
+ p3s_pred_entry = "points_3d"
2190
+ print(f" Pred point map: {p3s_pred_path}")
2191
+ points_3d_sparse_dataset = build_points_3d_sparse_eval_dataset(
2192
+ gt_sparse_depth_path=gt_sparse_depth_path,
2193
+ pred_points_3d_path=p3s_pred_path,
2194
+ intrinsics_path=intrinsics_path,
2195
+ camera_extrinsics_path=camera_extrinsics_path,
2196
+ lidar_extrinsics_path=lidar_extrinsics_path,
2197
+ segmentation_path=segmentation_path,
2198
+ segmentation_modality_key=segmentation_key or "segmentation",
2199
+ gt_sparse_depth_split=gt_sparse_depth_split,
2200
+ pred_points_3d_split=p3s_pred_split,
2201
+ intrinsics_split=intrinsics_split,
2202
+ camera_extrinsics_split=camera_extrinsics_split,
2203
+ lidar_extrinsics_split=lidar_extrinsics_split,
2204
+ segmentation_split=segmentation_split,
2205
+ )
2206
+ p3ds_meta = get_points_3d_metadata(points_3d_sparse_dataset)
2207
+ p3s_pred_is_radial = True
2208
+ p3s_alignment_mode = args.points_3d_alignment
2209
+ p3s_input_hint = None
2210
+ et_eval_datasets["points_3d_sparse"] = points_3d_sparse_dataset
2211
+ print(f" Matched pairs: {len(points_3d_sparse_dataset)}")
2212
+
2213
+ points_3d_sparse_results = evaluate_points_3d_sparse_samples(
2214
+ dataset=points_3d_sparse_dataset,
2215
+ pred_is_radial=p3s_pred_is_radial,
2216
+ gt_name=gt.get("name", "GT"),
2217
+ pred_name=ds_name,
2218
+ num_workers=args.num_workers,
2219
+ verbose=args.verbose,
2220
+ sanity_checker=sanity_checker,
2221
+ sky_mask_enabled=args.mask_sky,
2222
+ alignment_mode=p3s_alignment_mode,
2223
+ input_space_hint=p3s_input_hint,
2224
+ pred_is_depth=p3s_pred_is_depth,
2225
+ )
2226
+
2227
+ if sanity_checker is not None:
2228
+ sanity_checker.print_pair_report(ds_name, modality="points_3d")
2229
+
2230
+ p3s_space_info = points_3d_sparse_results.get("space_info", {})
2231
+ p3s_dataset_info = points_3d_sparse_results.get("dataset_info", {})
2232
+ p3s_spatial = points_3d_sparse_results.get("spatial_info", {})
2233
+
2234
+ points_3d_sparse_ns = _EvalNamespace(
2235
+ producer="euler-eval",
2236
+ producer_version=_get_version(),
2237
+ modalities=("points_3d",),
2238
+ axes=_points_3d_eval_axes(),
2239
+ descriptions=_POINTS_3D_EVAL_DESCRIPTIONS,
2240
+ )
2241
+ points_3d_sparse_save = {
2242
+ "metricSet": _points_3d_metric_set_envelope(
2243
+ points_3d_sparse_ns,
2244
+ metadata={
2245
+ "input_space_detected": p3s_space_info.get(
2246
+ "input_space_detected", "unknown"
2247
+ ),
2248
+ "metric_space_source": p3s_space_info.get(
2249
+ "metric_space_source"
2250
+ ),
2251
+ "calibration_mode": p3s_space_info.get(
2252
+ "calibration_mode", "unknown"
2253
+ ),
2254
+ "calibration_applied": p3s_space_info.get(
2255
+ "calibration_applied", False
2256
+ ),
2257
+ "emitted_spaces": p3s_space_info.get("emitted_spaces", []),
2258
+ "canonical_space": p3s_space_info.get(
2259
+ "canonical_space", "native"
2260
+ ),
2261
+ "gt_representation": "point_cloud",
2262
+ "fov_domain": p3s_dataset_info.get("fov_domain"),
2263
+ "threshold_deg": p3s_dataset_info.get("threshold_deg"),
2264
+ },
2265
+ ),
2266
+ "dataset_info": p3s_dataset_info,
2267
+ "meta": _clean_metric_tree({
2268
+ "version": _get_version(),
2269
+ "modality": "points_3d",
2270
+ "device": args.device,
2271
+ "gt": {
2272
+ "path": gt_sparse_depth_path,
2273
+ "split": gt_sparse_depth_split,
2274
+ "representation": "point_cloud",
2275
+ },
2276
+ "pred": {
2277
+ "path": p3s_pred_path,
2278
+ "split": p3s_pred_split,
2279
+ "entry": p3s_pred_entry,
2280
+ "representation": "depth" if p3s_pred_is_depth else "points_3d",
2281
+ "dimensions": p3s_spatial.get("pred_dimensions"),
2282
+ },
2283
+ "calibration": {
2284
+ "intrinsics_path": intrinsics_path,
2285
+ "intrinsics_split": intrinsics_split,
2286
+ "camera_extrinsics_path": camera_extrinsics_path,
2287
+ "camera_extrinsics_split": camera_extrinsics_split,
2288
+ "lidar_extrinsics_path": lidar_extrinsics_path,
2289
+ "lidar_extrinsics_split": lidar_extrinsics_split,
2290
+ },
2291
+ "spatial_alignment": {
2292
+ "method": p3s_spatial.get("method", "pointcloud_projection"),
2293
+ "evaluated_dimensions": p3s_spatial.get(
2294
+ "evaluated_dimensions"
2295
+ ),
2296
+ },
2297
+ "modality_params": p3ds_meta,
2298
+ "eval_params": {
2299
+ "sky_masking": args.mask_sky,
2300
+ "alignment_mode": p3s_alignment_mode,
2301
+ "num_workers": args.num_workers,
2302
+ },
2303
+ }),
2304
+ "points3d": {"eval": {}},
2305
+ }
2306
+ for space_name, result_key in (
2307
+ ("native", "points_3d_native"),
2308
+ ("metric", "points_3d_metric"),
2309
+ ):
2310
+ branch = points_3d_sparse_results.get(result_key)
2311
+ if branch is not None:
2312
+ points_3d_sparse_save["points3d"]["eval"][space_name] = (
2313
+ _clean_metric_tree(branch)
2314
+ )
2315
+ p3s_pfm = points_3d_sparse_results.get("per_file_metrics", {})
2316
+ if p3s_pfm:
2317
+ canonical_space = p3s_space_info.get("canonical_space", "native")
2318
+ points_3d_sparse_save["per_file_metrics"] = _clean_per_file_metrics(
2319
+ _wrap_pfm_metrics(
2320
+ p3s_pfm,
2321
+ lambda m: _wrap_depth_space_pfm_metrics(
2322
+ m,
2323
+ metric_root="points3d",
2324
+ canonical_key="points_3d",
2325
+ canonical_space=canonical_space,
2326
+ ),
2327
+ ),
2328
+ label="points_3d sparse per_file_metrics",
2329
+ )
2330
+ all_results.setdefault("per_file_metrics", {}).update(p3s_pfm)
2331
+
2332
+ print_results(
2333
+ {k: v for k, v in points_3d_sparse_results.items()
2334
+ if k not in ("per_file_metrics", "spatial_info")},
2335
+ f"POINTS_3D · SPARSE: {ds_name}",
2336
+ )
2337
+
2113
2338
  # Save per-modality results to respective dataset paths
2114
2339
  if depth_save:
2115
2340
  depth_out = save_results(
@@ -2134,6 +2359,19 @@ def main():
2134
2359
  points_3d_save, dataset_config, modality="points_3d"
2135
2360
  )
2136
2361
  print(f"\n Points-3D results saved to: {points_3d_out}")
2362
+ if points_3d_sparse_save:
2363
+ # Distinct file so it never clobbers the sparse-depth eval.json that
2364
+ # shares the same dense-depth prediction path / output_file.
2365
+ points_3d_sparse_out = save_results(
2366
+ points_3d_sparse_save,
2367
+ dataset_config,
2368
+ modality="points_3d",
2369
+ default_basename="points3d_eval.json",
2370
+ output_file_suffix="_points3d",
2371
+ )
2372
+ print(
2373
+ f"\n Points-3D (sparse GT) results saved to: {points_3d_sparse_out}"
2374
+ )
2137
2375
 
2138
2376
  # Log to euler_train
2139
2377
  if et_run is not None and et_eval_datasets: