euler-eval 2.16.0__tar.gz → 2.19.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_eval-2.16.0 → euler_eval-2.19.0}/PKG-INFO +74 -3
- {euler_eval-2.16.0 → euler_eval-2.19.0}/README.md +73 -2
- euler_eval-2.19.0/euler_eval/__init__.py +65 -0
- euler_eval-2.19.0/euler_eval/calibration.py +258 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/cli.py +3 -22
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/evaluate.py +8 -221
- euler_eval-2.19.0/euler_eval/validation.py +593 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval.egg-info/PKG-INFO +74 -3
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval.egg-info/SOURCES.txt +4 -1
- {euler_eval-2.16.0 → euler_eval-2.19.0}/pyproject.toml +1 -1
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_points_3d_sparse.py +27 -19
- euler_eval-2.19.0/tests/test_validation.py +417 -0
- euler_eval-2.16.0/euler_eval/__init__.py +0 -1
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/config_paths.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/data.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/__init__.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/absrel.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/daniel_error.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/depth_binned_error.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/depth_edge_f1.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/depth_standard.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/fid_kid.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/gpu_depth_batch.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/gpu_image_batch.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/high_freq_energy.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/lpips_metric.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/normal_consistency.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/points3d_cloud.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/points3d_decomposition.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/points3d_distance.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/points3d_geometry.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/psnr.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/rgb_edge_f1.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/rgb_lpips.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/rgb_psnr_ssim.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/rho_a.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/rmse.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/scale_invariant_log.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/ssim.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/tail_errors.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/metrics/utils.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/sanity_checker.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval/utils/hierarchy_parser.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval.egg-info/dependency_links.txt +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval.egg-info/entry_points.txt +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval.egg-info/requires.txt +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/euler_eval.egg-info/top_level.txt +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/init_cache.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/setup.cfg +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_alignment.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_cli_device.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_config.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_data.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_data_builders.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_depth_alignment_output.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_depth_standard.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_evaluate_helpers.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_init_cache.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_integration.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_meta_output.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_metrics_utils.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_points_3d.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_rgb_fid_output.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_rho_a.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_save_results.py +0 -0
- {euler_eval-2.16.0 → euler_eval-2.19.0}/tests/test_sparse_depth.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: euler-eval
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.19.0
|
|
4
4
|
Summary: Depth map evaluation toolkit with comprehensive metrics
|
|
5
5
|
Author: Depth Eval Contributors
|
|
6
6
|
License: MIT
|
|
@@ -221,6 +221,77 @@ The interval bounds are `near=[MIN, near_max)`, `mid=[near_max, mid_max)`, and
|
|
|
221
221
|
| `mid` | `[9.290856529, 35.954189863)` |
|
|
222
222
|
| `far` | `[35.954189863, 80.0]` |
|
|
223
223
|
|
|
224
|
+
## Programmatic use (in-training validation)
|
|
225
|
+
|
|
226
|
+
Besides the file-based CLI, `euler_eval.validation` exposes the same depth
|
|
227
|
+
metric semantics for **in-memory predictions** — e.g. scoring a training
|
|
228
|
+
run's validation pass against dense *or sparse* GT without writing
|
|
229
|
+
predictions to disk:
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
from euler_eval import (
|
|
233
|
+
DepthValidationAggregator,
|
|
234
|
+
build_validation_gt_dataset,
|
|
235
|
+
evaluate_dense_depth_sample,
|
|
236
|
+
evaluate_sparse_depth_sample,
|
|
237
|
+
get_sample_intrinsics,
|
|
238
|
+
get_sample_pointcloud_to_camera_extrinsics,
|
|
239
|
+
summarize_reduced_state,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# GT-only dataset from euler-loading compatible paths (.ds_crawler indices,
|
|
243
|
+
# inline :split#scope= selectors supported).
|
|
244
|
+
dataset = build_validation_gt_dataset(
|
|
245
|
+
sparse_depth_path="/data/muses.zip:val#scope=lidar",
|
|
246
|
+
rgb_path="/data/muses.zip:val#scope=rgb",
|
|
247
|
+
intrinsics_path="/data/muses.zip:val#scope=intrinsics",
|
|
248
|
+
camera_extrinsics_path="/data/muses.zip:val#scope=extrinsics",
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
aggregator = DepthValidationAggregator()
|
|
252
|
+
for i in range(len(dataset)):
|
|
253
|
+
sample = dataset[i]
|
|
254
|
+
depth_pred = my_model(sample["rgb"]) # (H, W) metres, planar z
|
|
255
|
+
K = get_sample_intrinsics(sample)
|
|
256
|
+
lidar2cam, _ = get_sample_pointcloud_to_camera_extrinsics(sample)
|
|
257
|
+
aggregator.update(
|
|
258
|
+
evaluate_sparse_depth_sample(
|
|
259
|
+
depth_pred,
|
|
260
|
+
sample["sparse_depth"], # (N, C>=3) lidar points
|
|
261
|
+
K,
|
|
262
|
+
lidar2cam,
|
|
263
|
+
alignment="none", # "affine" for relative depth
|
|
264
|
+
)
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
summary = aggregator.summary()
|
|
268
|
+
print(summary["standard"]["image_mean"]["absrel"])
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Key pieces:
|
|
272
|
+
|
|
273
|
+
- `evaluate_dense_depth_sample(pred, gt, valid_mask=None, alignment=...,
|
|
274
|
+
min_depth=..., max_depth=...)` — one dense prediction vs a dense GT map.
|
|
275
|
+
GT at another resolution is aligned to the prediction plane; returns the
|
|
276
|
+
standard metric set (`absrel`, `sqrel`, `mae`, `rmse`, `rmse_log`, `log10`,
|
|
277
|
+
`silog`, `delta1-3`) plus pooled pixel statistics, or `None` when too few
|
|
278
|
+
valid pixels remain.
|
|
279
|
+
- `evaluate_sparse_depth_sample(pred, point_cloud, intrinsics,
|
|
280
|
+
camera_extrinsics, lidar_extrinsics=None, pred_is_radial=False, ...)` —
|
|
281
|
+
projects the sparse GT cloud into the prediction plane (radial depth,
|
|
282
|
+
nearest-z occlusion handling), converts a planar prediction to radial with
|
|
283
|
+
the same intrinsics, and scores only the projected pixels — matching the
|
|
284
|
+
CLI's `gt.sparse_depth` pipeline. Pass the intrinsics of the *actual
|
|
285
|
+
prediction plane* (i.e. adjusted for any crop/resize of the model input).
|
|
286
|
+
- `alignment="none"` scores metric predictions as-is; `"affine"` fits
|
|
287
|
+
least-squares scale+shift first (relative/affine depth models).
|
|
288
|
+
- `DepthValidationAggregator` accumulates per-sample results into the CLI's
|
|
289
|
+
`image_mean` / `image_median` / `pixel_pool` reducers. For multi-process
|
|
290
|
+
validation, sum `aggregator.reduced_state()` vectors across ranks (fixed
|
|
291
|
+
key order via `DepthValidationAggregator.state_keys()`) and rebuild
|
|
292
|
+
mean-based summaries with `summarize_reduced_state(...)`.
|
|
293
|
+
- Inputs accept torch tensors or numpy arrays; all math runs on CPU numpy.
|
|
294
|
+
|
|
224
295
|
## Configuration
|
|
225
296
|
|
|
226
297
|
### `config.json`
|
|
@@ -268,7 +339,7 @@ When the GT is a sparse pointcloud, the evaluator also produces **3D (`points_3d
|
|
|
268
339
|
- a **predicted point map** (`datasets[].points_3d`, an `(H,W,3)` map) — evaluated *directly* against the sparse GT cloud, with the 3D **similarity** gauge selected by `--points-3d-alignment`; or
|
|
269
340
|
- a **dense depth map** (`datasets[].depth`/`relative_depth`/`affine_depth`) — unprojected with `gt.intrinsics` into a point map, with the depth **affine** gauge selected by `--depth-alignment`.
|
|
270
341
|
|
|
271
|
-
If a dataset entry provides both, the predicted `points_3d` map is preferred (the depth is still scored by the pointwise sparse-depth metrics). Either way the sparse GT cloud is projected into the prediction plane, and the 3D results are written
|
|
342
|
+
If a dataset entry provides both, the predicted `points_3d` map is preferred (the depth is still scored by the pointwise sparse-depth metrics). Either way the sparse GT cloud is projected into the prediction plane, and the 3D results are written as `eval.json` in the selected prediction modality path (or to the configured `output_file`). Give simultaneous evaluations distinct prediction paths: one `output_file` applies to the whole dataset entry.
|
|
272
343
|
|
|
273
344
|
Sparse depth does not require segmentation GT. `gt.segmentation` and
|
|
274
345
|
`gt.semantic_segmentation` are optional aliases for the same sky-mask source.
|
|
@@ -536,7 +607,7 @@ For sparse depth outputs, the internal Python result dict still uses `sparse_dep
|
|
|
536
607
|
|
|
537
608
|
For points_3d outputs, the internal Python result dict uses `points_3d_native`, `points_3d_metric`, and `points_3d` (canonical alias), but serialized `eval.json` metric paths are rooted at `points3d.eval` (no underscore) to satisfy the namespace first-segment rule, mirroring `sparsedepth`. The `metric` space is emitted only when a gauge alignment is applied; otherwise only `native` is present and aliased.
|
|
538
609
|
|
|
539
|
-
When the points_3d metrics come from a sparse pointcloud GT (`gt.sparse_depth` + a predicted `points_3d` map or a dense depth prediction), they use the same `points3d.eval` namespace
|
|
610
|
+
When the points_3d metrics come from a sparse pointcloud GT (`gt.sparse_depth` + a predicted `points_3d` map or a dense depth prediction), they use the same `points3d.eval` namespace and are written as `eval.json` beside the selected prediction modality (or to the configured `output_file`). Only the `point_error`, `error_decomposition`, and (directed) `cloud_distance` categories are present; the `metric` space is emitted only when the gauge (`--points-3d-alignment` for a point-map prediction, `--depth-alignment` for a depth prediction) calibrates the prediction.
|
|
540
611
|
|
|
541
612
|
Previous single-depth structure (kept under `depth`) is:
|
|
542
613
|
|
|
@@ -183,6 +183,77 @@ The interval bounds are `near=[MIN, near_max)`, `mid=[near_max, mid_max)`, and
|
|
|
183
183
|
| `mid` | `[9.290856529, 35.954189863)` |
|
|
184
184
|
| `far` | `[35.954189863, 80.0]` |
|
|
185
185
|
|
|
186
|
+
## Programmatic use (in-training validation)
|
|
187
|
+
|
|
188
|
+
Besides the file-based CLI, `euler_eval.validation` exposes the same depth
|
|
189
|
+
metric semantics for **in-memory predictions** — e.g. scoring a training
|
|
190
|
+
run's validation pass against dense *or sparse* GT without writing
|
|
191
|
+
predictions to disk:
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
from euler_eval import (
|
|
195
|
+
DepthValidationAggregator,
|
|
196
|
+
build_validation_gt_dataset,
|
|
197
|
+
evaluate_dense_depth_sample,
|
|
198
|
+
evaluate_sparse_depth_sample,
|
|
199
|
+
get_sample_intrinsics,
|
|
200
|
+
get_sample_pointcloud_to_camera_extrinsics,
|
|
201
|
+
summarize_reduced_state,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
# GT-only dataset from euler-loading compatible paths (.ds_crawler indices,
|
|
205
|
+
# inline :split#scope= selectors supported).
|
|
206
|
+
dataset = build_validation_gt_dataset(
|
|
207
|
+
sparse_depth_path="/data/muses.zip:val#scope=lidar",
|
|
208
|
+
rgb_path="/data/muses.zip:val#scope=rgb",
|
|
209
|
+
intrinsics_path="/data/muses.zip:val#scope=intrinsics",
|
|
210
|
+
camera_extrinsics_path="/data/muses.zip:val#scope=extrinsics",
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
aggregator = DepthValidationAggregator()
|
|
214
|
+
for i in range(len(dataset)):
|
|
215
|
+
sample = dataset[i]
|
|
216
|
+
depth_pred = my_model(sample["rgb"]) # (H, W) metres, planar z
|
|
217
|
+
K = get_sample_intrinsics(sample)
|
|
218
|
+
lidar2cam, _ = get_sample_pointcloud_to_camera_extrinsics(sample)
|
|
219
|
+
aggregator.update(
|
|
220
|
+
evaluate_sparse_depth_sample(
|
|
221
|
+
depth_pred,
|
|
222
|
+
sample["sparse_depth"], # (N, C>=3) lidar points
|
|
223
|
+
K,
|
|
224
|
+
lidar2cam,
|
|
225
|
+
alignment="none", # "affine" for relative depth
|
|
226
|
+
)
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
summary = aggregator.summary()
|
|
230
|
+
print(summary["standard"]["image_mean"]["absrel"])
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
Key pieces:
|
|
234
|
+
|
|
235
|
+
- `evaluate_dense_depth_sample(pred, gt, valid_mask=None, alignment=...,
|
|
236
|
+
min_depth=..., max_depth=...)` — one dense prediction vs a dense GT map.
|
|
237
|
+
GT at another resolution is aligned to the prediction plane; returns the
|
|
238
|
+
standard metric set (`absrel`, `sqrel`, `mae`, `rmse`, `rmse_log`, `log10`,
|
|
239
|
+
`silog`, `delta1-3`) plus pooled pixel statistics, or `None` when too few
|
|
240
|
+
valid pixels remain.
|
|
241
|
+
- `evaluate_sparse_depth_sample(pred, point_cloud, intrinsics,
|
|
242
|
+
camera_extrinsics, lidar_extrinsics=None, pred_is_radial=False, ...)` —
|
|
243
|
+
projects the sparse GT cloud into the prediction plane (radial depth,
|
|
244
|
+
nearest-z occlusion handling), converts a planar prediction to radial with
|
|
245
|
+
the same intrinsics, and scores only the projected pixels — matching the
|
|
246
|
+
CLI's `gt.sparse_depth` pipeline. Pass the intrinsics of the *actual
|
|
247
|
+
prediction plane* (i.e. adjusted for any crop/resize of the model input).
|
|
248
|
+
- `alignment="none"` scores metric predictions as-is; `"affine"` fits
|
|
249
|
+
least-squares scale+shift first (relative/affine depth models).
|
|
250
|
+
- `DepthValidationAggregator` accumulates per-sample results into the CLI's
|
|
251
|
+
`image_mean` / `image_median` / `pixel_pool` reducers. For multi-process
|
|
252
|
+
validation, sum `aggregator.reduced_state()` vectors across ranks (fixed
|
|
253
|
+
key order via `DepthValidationAggregator.state_keys()`) and rebuild
|
|
254
|
+
mean-based summaries with `summarize_reduced_state(...)`.
|
|
255
|
+
- Inputs accept torch tensors or numpy arrays; all math runs on CPU numpy.
|
|
256
|
+
|
|
186
257
|
## Configuration
|
|
187
258
|
|
|
188
259
|
### `config.json`
|
|
@@ -230,7 +301,7 @@ When the GT is a sparse pointcloud, the evaluator also produces **3D (`points_3d
|
|
|
230
301
|
- a **predicted point map** (`datasets[].points_3d`, an `(H,W,3)` map) — evaluated *directly* against the sparse GT cloud, with the 3D **similarity** gauge selected by `--points-3d-alignment`; or
|
|
231
302
|
- a **dense depth map** (`datasets[].depth`/`relative_depth`/`affine_depth`) — unprojected with `gt.intrinsics` into a point map, with the depth **affine** gauge selected by `--depth-alignment`.
|
|
232
303
|
|
|
233
|
-
If a dataset entry provides both, the predicted `points_3d` map is preferred (the depth is still scored by the pointwise sparse-depth metrics). Either way the sparse GT cloud is projected into the prediction plane, and the 3D results are written
|
|
304
|
+
If a dataset entry provides both, the predicted `points_3d` map is preferred (the depth is still scored by the pointwise sparse-depth metrics). Either way the sparse GT cloud is projected into the prediction plane, and the 3D results are written as `eval.json` in the selected prediction modality path (or to the configured `output_file`). Give simultaneous evaluations distinct prediction paths: one `output_file` applies to the whole dataset entry.
|
|
234
305
|
|
|
235
306
|
Sparse depth does not require segmentation GT. `gt.segmentation` and
|
|
236
307
|
`gt.semantic_segmentation` are optional aliases for the same sky-mask source.
|
|
@@ -498,7 +569,7 @@ For sparse depth outputs, the internal Python result dict still uses `sparse_dep
|
|
|
498
569
|
|
|
499
570
|
For points_3d outputs, the internal Python result dict uses `points_3d_native`, `points_3d_metric`, and `points_3d` (canonical alias), but serialized `eval.json` metric paths are rooted at `points3d.eval` (no underscore) to satisfy the namespace first-segment rule, mirroring `sparsedepth`. The `metric` space is emitted only when a gauge alignment is applied; otherwise only `native` is present and aliased.
|
|
500
571
|
|
|
501
|
-
When the points_3d metrics come from a sparse pointcloud GT (`gt.sparse_depth` + a predicted `points_3d` map or a dense depth prediction), they use the same `points3d.eval` namespace
|
|
572
|
+
When the points_3d metrics come from a sparse pointcloud GT (`gt.sparse_depth` + a predicted `points_3d` map or a dense depth prediction), they use the same `points3d.eval` namespace and are written as `eval.json` beside the selected prediction modality (or to the configured `output_file`). Only the `point_error`, `error_decomposition`, and (directed) `cloud_distance` categories are present; the `metric` space is emitted only when the gauge (`--points-3d-alignment` for a point-map prediction, `--depth-alignment` for a depth prediction) calibrates the prediction.
|
|
502
573
|
|
|
503
574
|
Previous single-depth structure (kept under `depth`) is:
|
|
504
575
|
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Depth evaluation package.
|
|
2
|
+
|
|
3
|
+
Public programmatic API (lazily imported so that ``import euler_eval`` stays
|
|
4
|
+
cheap; heavy metric backends only load on first use):
|
|
5
|
+
|
|
6
|
+
* :mod:`euler_eval.validation` — in-training/programmatic depth validation
|
|
7
|
+
(``evaluate_dense_depth_sample``, ``evaluate_sparse_depth_sample``,
|
|
8
|
+
``DepthValidationAggregator``, ``build_validation_gt_dataset``).
|
|
9
|
+
* :mod:`euler_eval.calibration` — calibration extraction from euler-loading
|
|
10
|
+
samples.
|
|
11
|
+
* :mod:`euler_eval.metrics` — pure-array metric functions.
|
|
12
|
+
* :mod:`euler_eval.data` — dataset builders, converters, projection, and
|
|
13
|
+
alignment helpers.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
|
|
20
|
+
_VALIDATION_EXPORTS = (
|
|
21
|
+
"DepthSampleEvaluation",
|
|
22
|
+
"DepthValidationAggregator",
|
|
23
|
+
"VALIDATION_ALIGNMENT_MODES",
|
|
24
|
+
"build_validation_gt_dataset",
|
|
25
|
+
"evaluate_dense_depth_sample",
|
|
26
|
+
"evaluate_sparse_depth_sample",
|
|
27
|
+
"summarize_reduced_state",
|
|
28
|
+
)
|
|
29
|
+
_CALIBRATION_EXPORTS = (
|
|
30
|
+
"get_sample_intrinsics",
|
|
31
|
+
"get_sample_pointcloud_to_camera_extrinsics",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__all__ = [*_VALIDATION_EXPORTS, *_CALIBRATION_EXPORTS]
|
|
35
|
+
|
|
36
|
+
if TYPE_CHECKING: # pragma: no cover - static import surface for type checkers
|
|
37
|
+
from .calibration import ( # noqa: F401
|
|
38
|
+
get_sample_intrinsics,
|
|
39
|
+
get_sample_pointcloud_to_camera_extrinsics,
|
|
40
|
+
)
|
|
41
|
+
from .validation import ( # noqa: F401
|
|
42
|
+
VALIDATION_ALIGNMENT_MODES,
|
|
43
|
+
DepthSampleEvaluation,
|
|
44
|
+
DepthValidationAggregator,
|
|
45
|
+
build_validation_gt_dataset,
|
|
46
|
+
evaluate_dense_depth_sample,
|
|
47
|
+
evaluate_sparse_depth_sample,
|
|
48
|
+
summarize_reduced_state,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def __getattr__(name: str):
|
|
53
|
+
if name in _VALIDATION_EXPORTS:
|
|
54
|
+
from . import validation
|
|
55
|
+
|
|
56
|
+
return getattr(validation, name)
|
|
57
|
+
if name in _CALIBRATION_EXPORTS:
|
|
58
|
+
from . import calibration
|
|
59
|
+
|
|
60
|
+
return getattr(calibration, name)
|
|
61
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def __dir__() -> list[str]:
|
|
65
|
+
return sorted(set(globals()) | set(__all__))
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Camera-calibration extraction from euler-loading samples.
|
|
2
|
+
|
|
3
|
+
MultiModalDataset samples carry calibration either directly (a ``(3, 3)`` /
|
|
4
|
+
``(4, 4)`` tensor per sample) or wrapped in hierarchical-modality dicts and
|
|
5
|
+
nested calibration payloads (e.g. a MUSES ``calib.json`` with named
|
|
6
|
+
``lidar2rgb`` transforms). This module owns the tolerant extraction logic
|
|
7
|
+
shared by the CLI evaluators (:mod:`euler_eval.evaluate`) and the
|
|
8
|
+
programmatic validation interface (:mod:`euler_eval.validation`).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from .data import (
|
|
19
|
+
compose_sensor_to_camera_extrinsics,
|
|
20
|
+
to_numpy_extrinsics,
|
|
21
|
+
to_numpy_intrinsics,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"get_first_hierarchical_value",
|
|
26
|
+
"iter_hierarchical_values",
|
|
27
|
+
"get_sample_intrinsics",
|
|
28
|
+
"get_sample_pointcloud_to_camera_extrinsics",
|
|
29
|
+
"extract_wrapped_extrinsics",
|
|
30
|
+
"extract_named_extrinsics",
|
|
31
|
+
"extract_direct_source_to_camera",
|
|
32
|
+
"DIRECT_SOURCE_TO_CAMERA_KEYS",
|
|
33
|
+
"SOURCE_SENSOR_POSE_KEYS",
|
|
34
|
+
"CAMERA_SENSOR_POSE_KEYS",
|
|
35
|
+
"MATRIX_WRAPPER_KEYS",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def get_first_hierarchical_value(sample: dict, key: str):
|
|
40
|
+
"""Return ``sample[key]``, unwrapping hierarchical ``{file_id: value}`` dicts."""
|
|
41
|
+
data = sample.get(key)
|
|
42
|
+
if data is None:
|
|
43
|
+
return None
|
|
44
|
+
if isinstance(data, dict):
|
|
45
|
+
if not data:
|
|
46
|
+
return None
|
|
47
|
+
return next(iter(data.values()))
|
|
48
|
+
return data
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def iter_hierarchical_values(sample: dict, key: str) -> list:
|
|
52
|
+
"""Return candidate values for ``sample[key]`` (dict itself plus entries)."""
|
|
53
|
+
data = sample.get(key)
|
|
54
|
+
if data is None:
|
|
55
|
+
return []
|
|
56
|
+
if isinstance(data, dict):
|
|
57
|
+
return [data, *data.values()]
|
|
58
|
+
return [data]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def get_sample_intrinsics(sample: dict) -> Optional[np.ndarray]:
|
|
62
|
+
"""Extract a ``(3, 3)`` intrinsics matrix from sample calibration data."""
|
|
63
|
+
K_data = get_first_hierarchical_value(sample, "intrinsics")
|
|
64
|
+
if K_data is None:
|
|
65
|
+
K_data = get_first_hierarchical_value(sample, "calibration")
|
|
66
|
+
if K_data is None:
|
|
67
|
+
return None
|
|
68
|
+
return to_numpy_intrinsics(K_data)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
DIRECT_SOURCE_TO_CAMERA_KEYS = (
|
|
72
|
+
"lidar2rgb",
|
|
73
|
+
"lidar2camera",
|
|
74
|
+
"lidar2cam",
|
|
75
|
+
"lidar_to_rgb",
|
|
76
|
+
"lidar_to_camera",
|
|
77
|
+
"lidar_to_cam",
|
|
78
|
+
"source2rgb",
|
|
79
|
+
"source2camera",
|
|
80
|
+
"source_to_rgb",
|
|
81
|
+
"source_to_camera",
|
|
82
|
+
"sensor2rgb",
|
|
83
|
+
"sensor2camera",
|
|
84
|
+
"sensor_to_rgb",
|
|
85
|
+
"sensor_to_camera",
|
|
86
|
+
)
|
|
87
|
+
SOURCE_SENSOR_POSE_KEYS = (
|
|
88
|
+
"lidar",
|
|
89
|
+
"lidar_pose",
|
|
90
|
+
"lidar_extrinsics",
|
|
91
|
+
"source",
|
|
92
|
+
"source_sensor",
|
|
93
|
+
"source_extrinsics",
|
|
94
|
+
"sensor",
|
|
95
|
+
"sensor_extrinsics",
|
|
96
|
+
"lidar2gnss",
|
|
97
|
+
"lidar2ego",
|
|
98
|
+
"lidar2vehicle",
|
|
99
|
+
"lidar2world",
|
|
100
|
+
"lidar_to_gnss",
|
|
101
|
+
"lidar_to_ego",
|
|
102
|
+
"lidar_to_vehicle",
|
|
103
|
+
"lidar_to_world",
|
|
104
|
+
)
|
|
105
|
+
CAMERA_SENSOR_POSE_KEYS = (
|
|
106
|
+
"rgb",
|
|
107
|
+
"camera",
|
|
108
|
+
"cam",
|
|
109
|
+
"frame_camera",
|
|
110
|
+
"rgb_pose",
|
|
111
|
+
"camera_pose",
|
|
112
|
+
"camera_extrinsics",
|
|
113
|
+
"target",
|
|
114
|
+
"target_sensor",
|
|
115
|
+
"target_extrinsics",
|
|
116
|
+
"rgb2gnss",
|
|
117
|
+
"camera2gnss",
|
|
118
|
+
"cam2gnss",
|
|
119
|
+
"frame_camera2gnss",
|
|
120
|
+
"rgb2ego",
|
|
121
|
+
"camera2ego",
|
|
122
|
+
"cam2ego",
|
|
123
|
+
"frame_camera2ego",
|
|
124
|
+
"rgb2vehicle",
|
|
125
|
+
"camera2vehicle",
|
|
126
|
+
"cam2vehicle",
|
|
127
|
+
"frame_camera2vehicle",
|
|
128
|
+
"rgb2world",
|
|
129
|
+
"camera2world",
|
|
130
|
+
"cam2world",
|
|
131
|
+
"frame_camera2world",
|
|
132
|
+
)
|
|
133
|
+
MATRIX_WRAPPER_KEYS = ("T", "transform", "matrix", "extrinsics", "pose")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _as_extrinsics_or_none(data) -> Optional[np.ndarray]:
|
|
137
|
+
try:
|
|
138
|
+
return to_numpy_extrinsics(data)
|
|
139
|
+
except (TypeError, ValueError):
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def extract_wrapped_extrinsics(data) -> Optional[np.ndarray]:
|
|
144
|
+
"""Extract a ``(4, 4)`` transform from a value or common wrapper mappings."""
|
|
145
|
+
matrix = _as_extrinsics_or_none(data)
|
|
146
|
+
if matrix is not None:
|
|
147
|
+
return matrix
|
|
148
|
+
|
|
149
|
+
if not isinstance(data, Mapping):
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
for key in MATRIX_WRAPPER_KEYS:
|
|
153
|
+
if key in data:
|
|
154
|
+
matrix = extract_wrapped_extrinsics(data[key])
|
|
155
|
+
if matrix is not None:
|
|
156
|
+
return matrix
|
|
157
|
+
|
|
158
|
+
if len(data) == 1:
|
|
159
|
+
return extract_wrapped_extrinsics(next(iter(data.values())))
|
|
160
|
+
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def extract_named_extrinsics(data, names: tuple[str, ...]) -> Optional[np.ndarray]:
|
|
165
|
+
"""Extract a transform stored under one of ``names`` in a calibration payload."""
|
|
166
|
+
if not isinstance(data, Mapping):
|
|
167
|
+
return extract_wrapped_extrinsics(data)
|
|
168
|
+
|
|
169
|
+
for name in names:
|
|
170
|
+
if name in data:
|
|
171
|
+
matrix = extract_wrapped_extrinsics(data[name])
|
|
172
|
+
if matrix is not None:
|
|
173
|
+
return matrix
|
|
174
|
+
|
|
175
|
+
for key in ("extrinsics", "sensors", "calibration", "poses"):
|
|
176
|
+
if key in data and isinstance(data[key], Mapping):
|
|
177
|
+
matrix = extract_named_extrinsics(data[key], names)
|
|
178
|
+
if matrix is not None:
|
|
179
|
+
return matrix
|
|
180
|
+
|
|
181
|
+
if len(data) == 1:
|
|
182
|
+
return extract_named_extrinsics(next(iter(data.values())), names)
|
|
183
|
+
|
|
184
|
+
return None
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def extract_direct_source_to_camera(data) -> Optional[np.ndarray]:
|
|
188
|
+
"""Extract a direct source→camera transform (e.g. MUSES ``lidar2rgb``)."""
|
|
189
|
+
matrix = _as_extrinsics_or_none(data)
|
|
190
|
+
if matrix is not None:
|
|
191
|
+
return matrix
|
|
192
|
+
|
|
193
|
+
if not isinstance(data, Mapping):
|
|
194
|
+
return None
|
|
195
|
+
|
|
196
|
+
for key in DIRECT_SOURCE_TO_CAMERA_KEYS:
|
|
197
|
+
if key in data:
|
|
198
|
+
matrix = extract_wrapped_extrinsics(data[key])
|
|
199
|
+
if matrix is not None:
|
|
200
|
+
return matrix
|
|
201
|
+
|
|
202
|
+
for key in ("extrinsics", "calibration"):
|
|
203
|
+
if key in data and isinstance(data[key], Mapping):
|
|
204
|
+
matrix = extract_direct_source_to_camera(data[key])
|
|
205
|
+
if matrix is not None:
|
|
206
|
+
return matrix
|
|
207
|
+
|
|
208
|
+
if len(data) == 1:
|
|
209
|
+
return extract_direct_source_to_camera(next(iter(data.values())))
|
|
210
|
+
|
|
211
|
+
return None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def get_sample_pointcloud_to_camera_extrinsics(
|
|
215
|
+
sample: dict,
|
|
216
|
+
) -> tuple[Optional[np.ndarray], Optional[str]]:
|
|
217
|
+
"""Extract or compose the transform used for sparse point projection.
|
|
218
|
+
|
|
219
|
+
The common MUSES path supplies a direct ``lidar2rgb`` transform via
|
|
220
|
+
``camera_extrinsics``. Some datasets instead expose separate lidar and
|
|
221
|
+
camera sensor poses in a shared frame; in that case we compose them to the
|
|
222
|
+
same source-to-camera transform expected by the projector.
|
|
223
|
+
"""
|
|
224
|
+
lidar_pose = None
|
|
225
|
+
for value in iter_hierarchical_values(sample, "lidar_extrinsics"):
|
|
226
|
+
lidar_pose = extract_named_extrinsics(value, SOURCE_SENSOR_POSE_KEYS)
|
|
227
|
+
if lidar_pose is not None:
|
|
228
|
+
break
|
|
229
|
+
|
|
230
|
+
camera_values = iter_hierarchical_values(sample, "camera_extrinsics")
|
|
231
|
+
if not camera_values:
|
|
232
|
+
camera_values = iter_hierarchical_values(sample, "extrinsics")
|
|
233
|
+
|
|
234
|
+
if lidar_pose is not None:
|
|
235
|
+
for value in camera_values:
|
|
236
|
+
camera_pose = extract_named_extrinsics(value, CAMERA_SENSOR_POSE_KEYS)
|
|
237
|
+
if camera_pose is not None:
|
|
238
|
+
return (
|
|
239
|
+
compose_sensor_to_camera_extrinsics(lidar_pose, camera_pose),
|
|
240
|
+
"composed_lidar_and_camera_sensor_poses",
|
|
241
|
+
)
|
|
242
|
+
return None, None
|
|
243
|
+
|
|
244
|
+
for value in camera_values:
|
|
245
|
+
direct = extract_direct_source_to_camera(value)
|
|
246
|
+
if direct is not None:
|
|
247
|
+
return direct, "direct_source_to_camera"
|
|
248
|
+
|
|
249
|
+
for value in camera_values:
|
|
250
|
+
lidar_pose = extract_named_extrinsics(value, SOURCE_SENSOR_POSE_KEYS)
|
|
251
|
+
camera_pose = extract_named_extrinsics(value, CAMERA_SENSOR_POSE_KEYS)
|
|
252
|
+
if lidar_pose is not None and camera_pose is not None:
|
|
253
|
+
return (
|
|
254
|
+
compose_sensor_to_camera_extrinsics(lidar_pose, camera_pose),
|
|
255
|
+
"composed_lidar_and_camera_sensor_poses",
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
return None, None
|
|
@@ -978,9 +978,6 @@ def save_results(
|
|
|
978
978
|
results: dict,
|
|
979
979
|
dataset_config: dict,
|
|
980
980
|
modality: str | None = None,
|
|
981
|
-
*,
|
|
982
|
-
default_basename: str = "eval.json",
|
|
983
|
-
output_file_suffix: str | None = None,
|
|
984
981
|
) -> Path:
|
|
985
982
|
"""Save results to output file.
|
|
986
983
|
|
|
@@ -994,14 +991,6 @@ def save_results(
|
|
|
994
991
|
modality: When set, save to this specific modality's path
|
|
995
992
|
(e.g. ``"depth"`` or ``"rgb"``). Falls back to the first
|
|
996
993
|
available modality path when *None*.
|
|
997
|
-
default_basename: File name used when no explicit ``output_file`` is
|
|
998
|
-
configured (default ``"eval.json"``). Lets a second modality that
|
|
999
|
-
shares a prediction path (e.g. sparse ``points_3d`` alongside sparse
|
|
1000
|
-
``depth``) write to a distinct file instead of clobbering it.
|
|
1001
|
-
output_file_suffix: When an explicit ``output_file`` is configured, this
|
|
1002
|
-
suffix is inserted before its extension (e.g. ``"_points3d"`` turns
|
|
1003
|
-
``model_a.json`` into ``model_a_points3d.json``) so multiple
|
|
1004
|
-
modalities can honour a single configured path without collision.
|
|
1005
994
|
|
|
1006
995
|
Returns:
|
|
1007
996
|
Path where results were saved.
|
|
@@ -1021,7 +1010,7 @@ def save_results(
|
|
|
1021
1010
|
dataset_config[selected_modality]["path"],
|
|
1022
1011
|
split=dataset_config[selected_modality].get("split"),
|
|
1023
1012
|
)
|
|
1024
|
-
/
|
|
1013
|
+
/ "eval.json"
|
|
1025
1014
|
)
|
|
1026
1015
|
else:
|
|
1027
1016
|
# Default: save alongside first available modality path
|
|
@@ -1032,17 +1021,13 @@ def save_results(
|
|
|
1032
1021
|
dataset_config[mod]["path"],
|
|
1033
1022
|
split=dataset_config[mod].get("split"),
|
|
1034
1023
|
)
|
|
1035
|
-
/
|
|
1024
|
+
/ "eval.json"
|
|
1036
1025
|
)
|
|
1037
1026
|
break
|
|
1038
1027
|
if output_file is None:
|
|
1039
|
-
output_file = Path(
|
|
1028
|
+
output_file = Path("eval.json")
|
|
1040
1029
|
else:
|
|
1041
1030
|
output_file = Path(output_file)
|
|
1042
|
-
if output_file_suffix:
|
|
1043
|
-
output_file = output_file.with_name(
|
|
1044
|
-
f"{output_file.stem}{output_file_suffix}{output_file.suffix}"
|
|
1045
|
-
)
|
|
1046
1031
|
|
|
1047
1032
|
zip_path, internal_name = _find_zip_ancestor(output_file)
|
|
1048
1033
|
if zip_path is not None and internal_name:
|
|
@@ -2360,14 +2345,10 @@ def main():
|
|
|
2360
2345
|
)
|
|
2361
2346
|
print(f"\n Points-3D results saved to: {points_3d_out}")
|
|
2362
2347
|
if points_3d_sparse_save:
|
|
2363
|
-
# Distinct file so it never clobbers the sparse-depth eval.json that
|
|
2364
|
-
# shares the same dense-depth prediction path / output_file.
|
|
2365
2348
|
points_3d_sparse_out = save_results(
|
|
2366
2349
|
points_3d_sparse_save,
|
|
2367
2350
|
dataset_config,
|
|
2368
2351
|
modality="points_3d",
|
|
2369
|
-
default_basename="points3d_eval.json",
|
|
2370
|
-
output_file_suffix="_points3d",
|
|
2371
2352
|
)
|
|
2372
2353
|
print(
|
|
2373
2354
|
f"\n Points-3D (sparse GT) results saved to: {points_3d_sparse_out}"
|