euler-eval 2.12.0__tar.gz → 2.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_eval-2.12.0 → euler_eval-2.14.0}/PKG-INFO +70 -6
- {euler_eval-2.12.0 → euler_eval-2.14.0}/README.md +69 -5
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/cli.py +446 -11
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/data.py +367 -11
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/evaluate.py +813 -50
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/__init__.py +31 -0
- euler_eval-2.14.0/euler_eval/metrics/points3d_cloud.py +101 -0
- euler_eval-2.14.0/euler_eval/metrics/points3d_decomposition.py +49 -0
- euler_eval-2.14.0/euler_eval/metrics/points3d_distance.py +58 -0
- euler_eval-2.14.0/euler_eval/metrics/points3d_geometry.py +196 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/sanity_checker.py +176 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval.egg-info/PKG-INFO +70 -6
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval.egg-info/SOURCES.txt +5 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/pyproject.toml +1 -1
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_config.py +52 -1
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_data_builders.py +85 -1
- euler_eval-2.14.0/tests/test_points_3d.py +574 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_sparse_depth.py +104 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/__init__.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/config_paths.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/absrel.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/daniel_error.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/depth_binned_error.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/depth_edge_f1.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/depth_standard.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/fid_kid.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/gpu_depth_batch.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/gpu_image_batch.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/high_freq_energy.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/lpips_metric.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/normal_consistency.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/psnr.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/rgb_edge_f1.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/rgb_lpips.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/rgb_psnr_ssim.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/rho_a.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/rmse.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/scale_invariant_log.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/ssim.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/tail_errors.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/metrics/utils.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval/utils/hierarchy_parser.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval.egg-info/dependency_links.txt +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval.egg-info/entry_points.txt +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval.egg-info/requires.txt +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/euler_eval.egg-info/top_level.txt +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/init_cache.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/setup.cfg +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_alignment.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_cli_device.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_data.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_depth_alignment_output.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_depth_standard.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_evaluate_helpers.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_init_cache.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_integration.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_meta_output.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_metrics_utils.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_rgb_fid_output.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_rho_a.py +0 -0
- {euler_eval-2.12.0 → euler_eval-2.14.0}/tests/test_save_results.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: euler-eval
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.14.0
|
|
4
4
|
Summary: Depth map evaluation toolkit with comprehensive metrics
|
|
5
5
|
Author: Depth Eval Contributors
|
|
6
6
|
License: MIT
|
|
@@ -45,6 +45,7 @@ A comprehensive evaluation toolkit for comparing predicted depth maps, RGB image
|
|
|
45
45
|
- **Depth metrics**: PSNR, SSIM, LPIPS, FID, KID, AbsRel, RMSE, Scale-Invariant Log Error, Normal Consistency, Depth Edge F1
|
|
46
46
|
- **RGB metrics**: PSNR, SSIM, LPIPS, FID, SCE (Structural Chromatic Error), Edge F1, Tail Errors (p95/p99), High-Frequency Energy Ratio, Depth-Binned Photometric Error
|
|
47
47
|
- **Rays metrics**: ρ_A (AUC of angular accuracy curve), Angular Error statistics and threshold percentages
|
|
48
|
+
- **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces.
|
|
48
49
|
- **Benchmark binning**: Optional depth-range benchmark that subdivides metrics into square-root-scaled near/mid/far bins
|
|
49
50
|
- **Sanity checking**: Automatic validation of metric results against configurable thresholds, with detailed warning reports
|
|
50
51
|
- **Sky masking**: Optional exclusion of sky regions from metrics using GT segmentation
|
|
@@ -136,10 +137,12 @@ This pre-downloads:
|
|
|
136
137
|
| `--skip-depth` | flag | off | Skip depth evaluation |
|
|
137
138
|
| `--skip-rgb` | flag | off | Skip RGB evaluation |
|
|
138
139
|
| `--skip-rays` | flag | off | Skip rays (spherical direction map) evaluation |
|
|
140
|
+
| `--skip-points-3d` | flag | off | Skip points_3d (per-pixel 3D point map) evaluation |
|
|
139
141
|
| `--mask-sky` | flag | off | Mask sky regions from metrics using GT segmentation |
|
|
140
142
|
| `--no-sanity-check` | flag | off | Disable sanity checking of metric configurations |
|
|
141
143
|
| `--metrics-config` | `str` | auto-detect | Path to `metrics_config.json` for sanity checking |
|
|
142
144
|
| `--depth-alignment` | `{none,auto_affine,affine}` | `auto_affine` | Depth calibration mode; outputs are emitted in semantic `native`/`metric` spaces and `depth` aliases the canonical branch |
|
|
145
|
+
| `--points-3d-alignment` | `{none,scale,similarity,auto}` | `auto` | points_3d gauge alignment. `none` scores raw points (metric models); `scale` fits a global scalar; `similarity` fits a 7-DoF Umeyama transform (relative models); `auto` applies `similarity` only for declared-relative predictions, else `none`. Outputs `native`/`metric` spaces with a `points_3d` canonical alias |
|
|
143
146
|
| `--rgb-fid-backend` | `{builtin,clean-fid}` | `builtin` | RGB FID backend; `clean-fid` requires optional dependency |
|
|
144
147
|
| `--benchmark-depth-range` | `float float` | none | Depth range `[MIN, MAX]` in meters for benchmark evaluation; computes depth and RGB metrics for pixels within this range, subdivided into square-root-scaled near/mid/far bins (additive to regular metrics) |
|
|
145
148
|
|
|
@@ -149,7 +152,7 @@ This pre-downloads:
|
|
|
149
152
|
# Evaluate with default settings (auto-selects CUDA when available)
|
|
150
153
|
depth-eval config.json --batch-size 32
|
|
151
154
|
|
|
152
|
-
# Evaluate with sky masking enabled (requires gt.segmentation
|
|
155
|
+
# Evaluate with sky masking enabled (requires gt.segmentation or gt.semantic_segmentation)
|
|
153
156
|
depth-eval config.json --mask-sky -v
|
|
154
157
|
|
|
155
158
|
# Skip RGB evaluation, only evaluate depth
|
|
@@ -175,6 +178,12 @@ depth-eval example_sparse_depth_config.json --skip-rgb --skip-rays
|
|
|
175
178
|
|
|
176
179
|
# Skip rays evaluation
|
|
177
180
|
depth-eval config.json --skip-rays
|
|
181
|
+
|
|
182
|
+
# Evaluate per-pixel 3D point maps (predict-your-own-camera models)
|
|
183
|
+
depth-eval example_points_3d_config.json --skip-depth --skip-rgb --skip-rays
|
|
184
|
+
|
|
185
|
+
# Align relative point maps to GT with a similarity (Umeyama) gauge
|
|
186
|
+
depth-eval example_points_3d_config.json --points-3d-alignment similarity
|
|
178
187
|
```
|
|
179
188
|
|
|
180
189
|
### Benchmark Depth Bins
|
|
@@ -241,7 +250,13 @@ Defines GT modalities, prediction datasets to evaluate, and optional euler_train
|
|
|
241
250
|
|
|
242
251
|
Each modality entry can optionally include a `split` field to select a specific split from the dataset (e.g. `{ "path": "/data/gt/depth", "split": "test" }`). euler-loading inline selectors are also accepted in `path`, such as `/data/muses.zip:test` or `/data/muses.zip:test#scope=rgb`.
|
|
243
252
|
|
|
244
|
-
For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.depth`. The prediction uses a dense depth-like map under `datasets[].depth`, `datasets[].relative_depth`, or `datasets[].affine_depth`. The evaluator projects the sparse GT point cloud into the prediction image plane using `gt.intrinsics` and `gt.camera_extrinsics`, then computes pointwise depth metrics only at projected valid pixels.
|
|
253
|
+
For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.depth`. The prediction uses a dense depth-like map under `datasets[].depth`, `datasets[].relative_depth`, or `datasets[].affine_depth`. The evaluator projects the sparse GT point cloud into the prediction image plane using `gt.intrinsics` and `gt.camera_extrinsics`, then computes pointwise depth metrics only at projected valid pixels. For MUSES through `euler_loading.loaders.muses`, `gt.camera_extrinsics` normally resolves to the direct `lidar2rgb` transform and no extra lidar pose is needed. If a dataset exposes separate lidar and camera poses in a shared frame instead, provide optional `gt.lidar_extrinsics`; the evaluator composes `inv(camera_pose) @ lidar_pose` before projection.
|
|
254
|
+
|
|
255
|
+
Sparse depth does not require segmentation GT. `gt.segmentation` and
|
|
256
|
+
`gt.semantic_segmentation` are optional aliases for the same sky-mask source.
|
|
257
|
+
They are loaded only when `--mask-sky` is set; in sparse depth evaluation that
|
|
258
|
+
mask excludes sky pixels from projected-point metrics and from scale/shift
|
|
259
|
+
fitting. Without `--mask-sky`, the segmentation entry is not loaded or used.
|
|
245
260
|
|
|
246
261
|
#### GT section
|
|
247
262
|
|
|
@@ -251,13 +266,15 @@ For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.dep
|
|
|
251
266
|
| `gt.depth.path` | no\* | Path to GT depth dataset |
|
|
252
267
|
| `gt.sparse_depth.path` | no\* | Path to sparse pointcloud GT dataset, e.g. `sparse_depth` with `(N,C)` points whose first columns are `x,y,z` in meters |
|
|
253
268
|
| `gt.rays.path` | no\* | Path to GT ray direction map dataset (for rays evaluation) |
|
|
254
|
-
| `gt.
|
|
269
|
+
| `gt.points_3d.path` | no\* | Path to GT per-pixel 3D point map dataset `(H,W,3)` in camera-frame metres (for points_3d evaluation) |
|
|
270
|
+
| `gt.segmentation.path` / `gt.semantic_segmentation.path` | no | Path to GT segmentation (needed for `--mask-sky`; use one key, not both) |
|
|
255
271
|
| `gt.calibration.path` | no | Path to calibration data (camera intrinsics matrices) |
|
|
256
272
|
| `gt.intrinsics.path` | required with `gt.sparse_depth` | Path to camera intrinsics matrices for pointcloud projection |
|
|
257
273
|
| `gt.camera_extrinsics.path` | required with `gt.sparse_depth` | Path to source-to-camera extrinsics, e.g. MUSES `lidar2rgb`, for pointcloud projection |
|
|
274
|
+
| `gt.lidar_extrinsics.path` | no | Optional lidar sensor pose in the same shared frame as `gt.camera_extrinsics`; when provided, `gt.camera_extrinsics` is interpreted as the camera pose and both are composed for projection |
|
|
258
275
|
| `gt.name` | no | Display name for ground truth (default: `"GT"`) |
|
|
259
276
|
|
|
260
|
-
\* At least one of `gt.rgb.path`, `gt.depth.path`, `gt.sparse_depth.path`, or `gt.
|
|
277
|
+
\* At least one of `gt.rgb.path`, `gt.depth.path`, `gt.sparse_depth.path`, `gt.rays.path`, or `gt.points_3d.path` is required.
|
|
261
278
|
|
|
262
279
|
#### Prediction datasets
|
|
263
280
|
|
|
@@ -271,9 +288,17 @@ Each entry in `datasets` can include `rgb`, one dense depth-like prediction (`de
|
|
|
271
288
|
| `relative_depth.path` | no\* | Path to predicted dense relative depth dataset; evaluated through the same depth pipeline with scale/shift alignment support |
|
|
272
289
|
| `affine_depth.path` | no\* | Path to predicted dense affine-depth dataset; evaluated through the same depth pipeline with scale/shift alignment support |
|
|
273
290
|
| `rays.path` | no\* | Path to predicted ray direction map dataset |
|
|
291
|
+
| `points_3d.path` | no\* | Path to predicted per-pixel 3D point map dataset `(H,W,3)` |
|
|
274
292
|
| `output_file` | no | Custom output path for results JSON (default: `eval.json` inside the first available modality path) |
|
|
275
293
|
|
|
276
|
-
\* At least one of `rgb.path`, `depth.path`, `relative_depth.path`, `affine_depth.path`, or `
|
|
294
|
+
\* At least one of `rgb.path`, `depth.path`, `relative_depth.path`, `affine_depth.path`, `rays.path`, or `points_3d.path` is required. Use only one dense depth-like entry per prediction dataset.
|
|
295
|
+
|
|
296
|
+
When evaluating a `points_3d` prediction, the GT point map comes from one of two sources:
|
|
297
|
+
|
|
298
|
+
1. **Explicit GT point map** — `gt.points_3d.path` (a stored `(H,W,3)` map).
|
|
299
|
+
2. **Synthesized from depth (fallback)** — if `gt.points_3d` is absent but `gt.depth.path` and intrinsics (`gt.intrinsics.path` or `gt.calibration.path`) are present, the evaluator unprojects GT depth with the intrinsics into a GT point map on the fly. This lets you score a `points_3d` prediction against a conventional depth + intrinsics ground truth without precomputing point maps. Whether the GT depth is radial or planar is read from its dataset metadata (`radial_depth`).
|
|
300
|
+
|
|
301
|
+
GT intrinsics under `gt.calibration` (or `gt.intrinsics`) are also used to auto-detect the FoV domain for the angular ρ_A sub-metric; for the explicit-point-map path they are optional and default to the `lfov` threshold when absent, but for the depth-synthesis fallback they are required. The gauge alignment is controlled by `--points-3d-alignment` (see [Points-3D gauge alignment](#points-3d-gauge-alignment)).
|
|
277
302
|
|
|
278
303
|
When `relative_depth` or `affine_depth` is used, `--depth-alignment auto_affine` treats the entry as declared non-metric depth and runs scale/shift alignment even if the raw value range is not normalized. `--depth-alignment none` still disables calibration.
|
|
279
304
|
|
|
@@ -367,6 +392,37 @@ Serialized `eval.json` sparse-depth metrics use the namespace root `sparsedepth`
|
|
|
367
392
|
| Angular Error | `rays.angular_error.mean_angle`, `median_angle` | Per-pixel angular error between predicted and GT camera ray directions (degrees) |
|
|
368
393
|
| Angular Error Thresholds | `rays.angular_error.percent_below_*` | Percentage of pixels with angular error below 5°, 10°, 15°, 20°, 30° |
|
|
369
394
|
|
|
395
|
+
### Points-3D metrics
|
|
396
|
+
|
|
397
|
+
For per-pixel 3D point maps `P(u,v) = (X,Y,Z)` in camera-frame metres. Metrics are
|
|
398
|
+
emitted in two gauge spaces (`native` raw prediction, `metric` after alignment) with
|
|
399
|
+
a `points_3d` canonical alias, keyed as
|
|
400
|
+
`points3d.eval.{native,metric}.{category}…`. Categories:
|
|
401
|
+
|
|
402
|
+
| Category | Key (per space) | Description |
|
|
403
|
+
|---|---|---|
|
|
404
|
+
| Euclidean 3D agreement | `point_error.{image_mean,image_median,pixel_pool}.*` | 3D EPE `mae3d`, `rmse3d`, `median3d`/`p90`/`p95`, relative error `rel_median`/`rel_p90`, and 3D δ-accuracy `acc_<τ>` (within τ m) / `acc_rel_<τ>` |
|
|
405
|
+
| Error decomposition | `error_decomposition.{radial_*,lateral_*,lateral_fraction}` + `angular_error.*` + `rho_a.*` | Splits each error vector into a **radial** (≈ depth) and **lateral** (≈ camera-model) component; `lateral_fraction ∈ [0,1]` attributes blame. Angular ray error and ρ_A (on native directions) measure the implied camera model |
|
|
406
|
+
| Geometric | `geometric.normal_consistency.*`, `geometric.point_edge_f1.*` | True-3D surface normals (cross-product of point-map tangents, no focal assumption) and a 3D discontinuity F1 |
|
|
407
|
+
| Cloud distance | `cloud_distance.chamfer.*`, `cloud_distance.fscore.tau_<τ>.*` | Correspondence-free Chamfer (accuracy/completeness/distance/median) and F-score (precision/recall/f1) at distance thresholds τ |
|
|
408
|
+
|
|
409
|
+
#### Points-3D gauge alignment
|
|
410
|
+
|
|
411
|
+
Point maps from different model families live in different frames, so `points_3d`
|
|
412
|
+
resolves an unknown **similarity gauge** (the 3D generalization of depth's
|
|
413
|
+
scale-and-shift) before the `metric`-space comparison:
|
|
414
|
+
|
|
415
|
+
- `none` — score the raw points as-is (metric, predict-your-own-camera models).
|
|
416
|
+
- `scale` — fit a single global scalar `s` (least squares).
|
|
417
|
+
- `similarity` — fit a 7-DoF Umeyama transform `s·R·p + t` over the known per-pixel
|
|
418
|
+
correspondences (relative / free-frame models).
|
|
419
|
+
- `auto` (default) — apply `similarity` for declared-relative predictions, else
|
|
420
|
+
`none`.
|
|
421
|
+
|
|
422
|
+
The angular `error_decomposition` ray metrics and `rho_a` are computed on the
|
|
423
|
+
**native** point-map directions (the camera-faithful frame); the radial/lateral and
|
|
424
|
+
Euclidean metrics are reported per space.
|
|
425
|
+
|
|
370
426
|
## Output
|
|
371
427
|
|
|
372
428
|
Results are saved as JSON per modality per prediction dataset (one file for depth or sparse depth, one for RGB, one for rays). Default path: `eval.json` inside each modality's dataset path, unless overridden by `output_file` in the config.
|
|
@@ -392,6 +448,12 @@ When `--rgb-fid-backend clean-fid` is used, `euler-eval` will honor `CLEANFID_CA
|
|
|
392
448
|
"sparsedepth": {
|
|
393
449
|
"eval": { "...": "serialized sparse-depth metrics when gt.sparse_depth is used" }
|
|
394
450
|
},
|
|
451
|
+
"points3d": {
|
|
452
|
+
"eval": {
|
|
453
|
+
"native": { "...": "raw-prediction point-map metrics" },
|
|
454
|
+
"metric": { "...": "gauge-aligned point-map metrics, when alignment ran" }
|
|
455
|
+
}
|
|
456
|
+
},
|
|
395
457
|
"rgb": {
|
|
396
458
|
"...": "..."
|
|
397
459
|
},
|
|
@@ -429,6 +491,8 @@ For depth outputs:
|
|
|
429
491
|
|
|
430
492
|
For sparse depth outputs, the internal Python result dict still uses `sparse_depth_native`, `sparse_depth_metric`, and `sparse_depth`, but serialized `eval.json` metric paths are rooted at `sparsedepth.eval` to satisfy external namespace validation. Only `standard` and `depth_metrics` categories are present.
|
|
431
493
|
|
|
494
|
+
For points_3d outputs, the internal Python result dict uses `points_3d_native`, `points_3d_metric`, and `points_3d` (canonical alias), but serialized `eval.json` metric paths are rooted at `points3d.eval` (no underscore) to satisfy the namespace first-segment rule, mirroring `sparsedepth`. The `metric` space is emitted only when a gauge alignment is applied; otherwise only `native` is present and aliased.
|
|
495
|
+
|
|
432
496
|
Previous single-depth structure (kept under `depth`) is:
|
|
433
497
|
|
|
434
498
|
```json
|
|
@@ -7,6 +7,7 @@ A comprehensive evaluation toolkit for comparing predicted depth maps, RGB image
|
|
|
7
7
|
- **Depth metrics**: PSNR, SSIM, LPIPS, FID, KID, AbsRel, RMSE, Scale-Invariant Log Error, Normal Consistency, Depth Edge F1
|
|
8
8
|
- **RGB metrics**: PSNR, SSIM, LPIPS, FID, SCE (Structural Chromatic Error), Edge F1, Tail Errors (p95/p99), High-Frequency Energy Ratio, Depth-Binned Photometric Error
|
|
9
9
|
- **Rays metrics**: ρ_A (AUC of angular accuracy curve), Angular Error statistics and threshold percentages
|
|
10
|
+
- **Points-3D metrics**: for models that predict a per-pixel 3D point map *and their own camera*: Euclidean 3D error (EPE/RMSE/3D δ-accuracy), radial-vs-lateral error decomposition (depth vs camera-model attribution) with ρ_A, true-3D surface normals, 3D edge F1, and correspondence-free Chamfer / F-score. Native + similarity-gauge-aligned (Umeyama) metric spaces.
|
|
10
11
|
- **Benchmark binning**: Optional depth-range benchmark that subdivides metrics into square-root-scaled near/mid/far bins
|
|
11
12
|
- **Sanity checking**: Automatic validation of metric results against configurable thresholds, with detailed warning reports
|
|
12
13
|
- **Sky masking**: Optional exclusion of sky regions from metrics using GT segmentation
|
|
@@ -98,10 +99,12 @@ This pre-downloads:
|
|
|
98
99
|
| `--skip-depth` | flag | off | Skip depth evaluation |
|
|
99
100
|
| `--skip-rgb` | flag | off | Skip RGB evaluation |
|
|
100
101
|
| `--skip-rays` | flag | off | Skip rays (spherical direction map) evaluation |
|
|
102
|
+
| `--skip-points-3d` | flag | off | Skip points_3d (per-pixel 3D point map) evaluation |
|
|
101
103
|
| `--mask-sky` | flag | off | Mask sky regions from metrics using GT segmentation |
|
|
102
104
|
| `--no-sanity-check` | flag | off | Disable sanity checking of metric configurations |
|
|
103
105
|
| `--metrics-config` | `str` | auto-detect | Path to `metrics_config.json` for sanity checking |
|
|
104
106
|
| `--depth-alignment` | `{none,auto_affine,affine}` | `auto_affine` | Depth calibration mode; outputs are emitted in semantic `native`/`metric` spaces and `depth` aliases the canonical branch |
|
|
107
|
+
| `--points-3d-alignment` | `{none,scale,similarity,auto}` | `auto` | points_3d gauge alignment. `none` scores raw points (metric models); `scale` fits a global scalar; `similarity` fits a 7-DoF Umeyama transform (relative models); `auto` applies `similarity` only for declared-relative predictions, else `none`. Outputs `native`/`metric` spaces with a `points_3d` canonical alias |
|
|
105
108
|
| `--rgb-fid-backend` | `{builtin,clean-fid}` | `builtin` | RGB FID backend; `clean-fid` requires optional dependency |
|
|
106
109
|
| `--benchmark-depth-range` | `float float` | none | Depth range `[MIN, MAX]` in meters for benchmark evaluation; computes depth and RGB metrics for pixels within this range, subdivided into square-root-scaled near/mid/far bins (additive to regular metrics) |
|
|
107
110
|
|
|
@@ -111,7 +114,7 @@ This pre-downloads:
|
|
|
111
114
|
# Evaluate with default settings (auto-selects CUDA when available)
|
|
112
115
|
depth-eval config.json --batch-size 32
|
|
113
116
|
|
|
114
|
-
# Evaluate with sky masking enabled (requires gt.segmentation
|
|
117
|
+
# Evaluate with sky masking enabled (requires gt.segmentation or gt.semantic_segmentation)
|
|
115
118
|
depth-eval config.json --mask-sky -v
|
|
116
119
|
|
|
117
120
|
# Skip RGB evaluation, only evaluate depth
|
|
@@ -137,6 +140,12 @@ depth-eval example_sparse_depth_config.json --skip-rgb --skip-rays
|
|
|
137
140
|
|
|
138
141
|
# Skip rays evaluation
|
|
139
142
|
depth-eval config.json --skip-rays
|
|
143
|
+
|
|
144
|
+
# Evaluate per-pixel 3D point maps (predict-your-own-camera models)
|
|
145
|
+
depth-eval example_points_3d_config.json --skip-depth --skip-rgb --skip-rays
|
|
146
|
+
|
|
147
|
+
# Align relative point maps to GT with a similarity (Umeyama) gauge
|
|
148
|
+
depth-eval example_points_3d_config.json --points-3d-alignment similarity
|
|
140
149
|
```
|
|
141
150
|
|
|
142
151
|
### Benchmark Depth Bins
|
|
@@ -203,7 +212,13 @@ Defines GT modalities, prediction datasets to evaluate, and optional euler_train
|
|
|
203
212
|
|
|
204
213
|
Each modality entry can optionally include a `split` field to select a specific split from the dataset (e.g. `{ "path": "/data/gt/depth", "split": "test" }`). euler-loading inline selectors are also accepted in `path`, such as `/data/muses.zip:test` or `/data/muses.zip:test#scope=rgb`.
|
|
205
214
|
|
|
206
|
-
For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.depth`. The prediction uses a dense depth-like map under `datasets[].depth`, `datasets[].relative_depth`, or `datasets[].affine_depth`. The evaluator projects the sparse GT point cloud into the prediction image plane using `gt.intrinsics` and `gt.camera_extrinsics`, then computes pointwise depth metrics only at projected valid pixels.
|
|
215
|
+
For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.depth`. The prediction uses a dense depth-like map under `datasets[].depth`, `datasets[].relative_depth`, or `datasets[].affine_depth`. The evaluator projects the sparse GT point cloud into the prediction image plane using `gt.intrinsics` and `gt.camera_extrinsics`, then computes pointwise depth metrics only at projected valid pixels. For MUSES through `euler_loading.loaders.muses`, `gt.camera_extrinsics` normally resolves to the direct `lidar2rgb` transform and no extra lidar pose is needed. If a dataset exposes separate lidar and camera poses in a shared frame instead, provide optional `gt.lidar_extrinsics`; the evaluator composes `inv(camera_pose) @ lidar_pose` before projection.
|
|
216
|
+
|
|
217
|
+
Sparse depth does not require segmentation GT. `gt.segmentation` and
|
|
218
|
+
`gt.semantic_segmentation` are optional aliases for the same sky-mask source.
|
|
219
|
+
They are loaded only when `--mask-sky` is set; in sparse depth evaluation that
|
|
220
|
+
mask excludes sky pixels from projected-point metrics and from scale/shift
|
|
221
|
+
fitting. Without `--mask-sky`, the segmentation entry is not loaded or used.
|
|
207
222
|
|
|
208
223
|
#### GT section
|
|
209
224
|
|
|
@@ -213,13 +228,15 @@ For sparse pointcloud depth evaluation, use `gt.sparse_depth` instead of `gt.dep
|
|
|
213
228
|
| `gt.depth.path` | no\* | Path to GT depth dataset |
|
|
214
229
|
| `gt.sparse_depth.path` | no\* | Path to sparse pointcloud GT dataset, e.g. `sparse_depth` with `(N,C)` points whose first columns are `x,y,z` in meters |
|
|
215
230
|
| `gt.rays.path` | no\* | Path to GT ray direction map dataset (for rays evaluation) |
|
|
216
|
-
| `gt.
|
|
231
|
+
| `gt.points_3d.path` | no\* | Path to GT per-pixel 3D point map dataset `(H,W,3)` in camera-frame metres (for points_3d evaluation) |
|
|
232
|
+
| `gt.segmentation.path` / `gt.semantic_segmentation.path` | no | Path to GT segmentation (needed for `--mask-sky`; use one key, not both) |
|
|
217
233
|
| `gt.calibration.path` | no | Path to calibration data (camera intrinsics matrices) |
|
|
218
234
|
| `gt.intrinsics.path` | required with `gt.sparse_depth` | Path to camera intrinsics matrices for pointcloud projection |
|
|
219
235
|
| `gt.camera_extrinsics.path` | required with `gt.sparse_depth` | Path to source-to-camera extrinsics, e.g. MUSES `lidar2rgb`, for pointcloud projection |
|
|
236
|
+
| `gt.lidar_extrinsics.path` | no | Optional lidar sensor pose in the same shared frame as `gt.camera_extrinsics`; when provided, `gt.camera_extrinsics` is interpreted as the camera pose and both are composed for projection |
|
|
220
237
|
| `gt.name` | no | Display name for ground truth (default: `"GT"`) |
|
|
221
238
|
|
|
222
|
-
\* At least one of `gt.rgb.path`, `gt.depth.path`, `gt.sparse_depth.path`, or `gt.
|
|
239
|
+
\* At least one of `gt.rgb.path`, `gt.depth.path`, `gt.sparse_depth.path`, `gt.rays.path`, or `gt.points_3d.path` is required.
|
|
223
240
|
|
|
224
241
|
#### Prediction datasets
|
|
225
242
|
|
|
@@ -233,9 +250,17 @@ Each entry in `datasets` can include `rgb`, one dense depth-like prediction (`de
|
|
|
233
250
|
| `relative_depth.path` | no\* | Path to predicted dense relative depth dataset; evaluated through the same depth pipeline with scale/shift alignment support |
|
|
234
251
|
| `affine_depth.path` | no\* | Path to predicted dense affine-depth dataset; evaluated through the same depth pipeline with scale/shift alignment support |
|
|
235
252
|
| `rays.path` | no\* | Path to predicted ray direction map dataset |
|
|
253
|
+
| `points_3d.path` | no\* | Path to predicted per-pixel 3D point map dataset `(H,W,3)` |
|
|
236
254
|
| `output_file` | no | Custom output path for results JSON (default: `eval.json` inside the first available modality path) |
|
|
237
255
|
|
|
238
|
-
\* At least one of `rgb.path`, `depth.path`, `relative_depth.path`, `affine_depth.path`, or `
|
|
256
|
+
\* At least one of `rgb.path`, `depth.path`, `relative_depth.path`, `affine_depth.path`, `rays.path`, or `points_3d.path` is required. Use only one dense depth-like entry per prediction dataset.
|
|
257
|
+
|
|
258
|
+
When evaluating a `points_3d` prediction, the GT point map comes from one of two sources:
|
|
259
|
+
|
|
260
|
+
1. **Explicit GT point map** — `gt.points_3d.path` (a stored `(H,W,3)` map).
|
|
261
|
+
2. **Synthesized from depth (fallback)** — if `gt.points_3d` is absent but `gt.depth.path` and intrinsics (`gt.intrinsics.path` or `gt.calibration.path`) are present, the evaluator unprojects GT depth with the intrinsics into a GT point map on the fly. This lets you score a `points_3d` prediction against a conventional depth + intrinsics ground truth without precomputing point maps. Whether the GT depth is radial or planar is read from its dataset metadata (`radial_depth`).
|
|
262
|
+
|
|
263
|
+
GT intrinsics under `gt.calibration` (or `gt.intrinsics`) are also used to auto-detect the FoV domain for the angular ρ_A sub-metric; for the explicit-point-map path they are optional and default to the `lfov` threshold when absent, but for the depth-synthesis fallback they are required. The gauge alignment is controlled by `--points-3d-alignment` (see [Points-3D gauge alignment](#points-3d-gauge-alignment)).
|
|
239
264
|
|
|
240
265
|
When `relative_depth` or `affine_depth` is used, `--depth-alignment auto_affine` treats the entry as declared non-metric depth and runs scale/shift alignment even if the raw value range is not normalized. `--depth-alignment none` still disables calibration.
|
|
241
266
|
|
|
@@ -329,6 +354,37 @@ Serialized `eval.json` sparse-depth metrics use the namespace root `sparsedepth`
|
|
|
329
354
|
| Angular Error | `rays.angular_error.mean_angle`, `median_angle` | Per-pixel angular error between predicted and GT camera ray directions (degrees) |
|
|
330
355
|
| Angular Error Thresholds | `rays.angular_error.percent_below_*` | Percentage of pixels with angular error below 5°, 10°, 15°, 20°, 30° |
|
|
331
356
|
|
|
357
|
+
### Points-3D metrics
|
|
358
|
+
|
|
359
|
+
For per-pixel 3D point maps `P(u,v) = (X,Y,Z)` in camera-frame metres. Metrics are
|
|
360
|
+
emitted in two gauge spaces (`native` raw prediction, `metric` after alignment) with
|
|
361
|
+
a `points_3d` canonical alias, keyed as
|
|
362
|
+
`points3d.eval.{native,metric}.{category}…`. Categories:
|
|
363
|
+
|
|
364
|
+
| Category | Key (per space) | Description |
|
|
365
|
+
|---|---|---|
|
|
366
|
+
| Euclidean 3D agreement | `point_error.{image_mean,image_median,pixel_pool}.*` | 3D EPE `mae3d`, `rmse3d`, `median3d`/`p90`/`p95`, relative error `rel_median`/`rel_p90`, and 3D δ-accuracy `acc_<τ>` (within τ m) / `acc_rel_<τ>` |
|
|
367
|
+
| Error decomposition | `error_decomposition.{radial_*,lateral_*,lateral_fraction}` + `angular_error.*` + `rho_a.*` | Splits each error vector into a **radial** (≈ depth) and **lateral** (≈ camera-model) component; `lateral_fraction ∈ [0,1]` attributes blame. Angular ray error and ρ_A (on native directions) measure the implied camera model |
|
|
368
|
+
| Geometric | `geometric.normal_consistency.*`, `geometric.point_edge_f1.*` | True-3D surface normals (cross-product of point-map tangents, no focal assumption) and a 3D discontinuity F1 |
|
|
369
|
+
| Cloud distance | `cloud_distance.chamfer.*`, `cloud_distance.fscore.tau_<τ>.*` | Correspondence-free Chamfer (accuracy/completeness/distance/median) and F-score (precision/recall/f1) at distance thresholds τ |
|
|
370
|
+
|
|
371
|
+
#### Points-3D gauge alignment
|
|
372
|
+
|
|
373
|
+
Point maps from different model families live in different frames, so `points_3d`
|
|
374
|
+
resolves an unknown **similarity gauge** (the 3D generalization of depth's
|
|
375
|
+
scale-and-shift) before the `metric`-space comparison:
|
|
376
|
+
|
|
377
|
+
- `none` — score the raw points as-is (metric, predict-your-own-camera models).
|
|
378
|
+
- `scale` — fit a single global scalar `s` (least squares).
|
|
379
|
+
- `similarity` — fit a 7-DoF Umeyama transform `s·R·p + t` over the known per-pixel
|
|
380
|
+
correspondences (relative / free-frame models).
|
|
381
|
+
- `auto` (default) — apply `similarity` for declared-relative predictions, else
|
|
382
|
+
`none`.
|
|
383
|
+
|
|
384
|
+
The angular `error_decomposition` ray metrics and `rho_a` are computed on the
|
|
385
|
+
**native** point-map directions (the camera-faithful frame); the radial/lateral and
|
|
386
|
+
Euclidean metrics are reported per space.
|
|
387
|
+
|
|
332
388
|
## Output
|
|
333
389
|
|
|
334
390
|
Results are saved as JSON per modality per prediction dataset (one file for depth or sparse depth, one for RGB, one for rays). Default path: `eval.json` inside each modality's dataset path, unless overridden by `output_file` in the config.
|
|
@@ -354,6 +410,12 @@ When `--rgb-fid-backend clean-fid` is used, `euler-eval` will honor `CLEANFID_CA
|
|
|
354
410
|
"sparsedepth": {
|
|
355
411
|
"eval": { "...": "serialized sparse-depth metrics when gt.sparse_depth is used" }
|
|
356
412
|
},
|
|
413
|
+
"points3d": {
|
|
414
|
+
"eval": {
|
|
415
|
+
"native": { "...": "raw-prediction point-map metrics" },
|
|
416
|
+
"metric": { "...": "gauge-aligned point-map metrics, when alignment ran" }
|
|
417
|
+
}
|
|
418
|
+
},
|
|
357
419
|
"rgb": {
|
|
358
420
|
"...": "..."
|
|
359
421
|
},
|
|
@@ -391,6 +453,8 @@ For depth outputs:
|
|
|
391
453
|
|
|
392
454
|
For sparse depth outputs, the internal Python result dict still uses `sparse_depth_native`, `sparse_depth_metric`, and `sparse_depth`, but serialized `eval.json` metric paths are rooted at `sparsedepth.eval` to satisfy external namespace validation. Only `standard` and `depth_metrics` categories are present.
|
|
393
455
|
|
|
456
|
+
For points_3d outputs, the internal Python result dict uses `points_3d_native`, `points_3d_metric`, and `points_3d` (canonical alias), but serialized `eval.json` metric paths are rooted at `points3d.eval` (no underscore) to satisfy the namespace first-segment rule, mirroring `sparsedepth`. The `metric` space is emitted only when a gauge alignment is applied; otherwise only `native` is present and aliased.
|
|
457
|
+
|
|
394
458
|
Previous single-depth structure (kept under `depth`) is:
|
|
395
459
|
|
|
396
460
|
```json
|