euler-eval 2.30.0__tar.gz → 2.31.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. {euler_eval-2.30.0/euler_eval.egg-info → euler_eval-2.31.0}/PKG-INFO +4 -3
  2. {euler_eval-2.30.0 → euler_eval-2.31.0}/README.md +2 -1
  3. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/alignment.md +2 -1
  4. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/cli.md +16 -0
  5. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/configuration.md +77 -14
  6. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/metrics.md +82 -0
  7. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/output.md +59 -0
  8. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/cli.py +430 -101
  9. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/config_paths.py +78 -2
  10. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/data.py +72 -46
  11. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/evaluate.py +359 -11
  12. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/__init__.py +18 -0
  13. euler_eval-2.31.0/euler_eval/metrics/sky_iou.py +167 -0
  14. {euler_eval-2.30.0 → euler_eval-2.31.0/euler_eval.egg-info}/PKG-INFO +4 -3
  15. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/SOURCES.txt +5 -0
  16. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/requires.txt +1 -1
  17. {euler_eval-2.30.0 → euler_eval-2.31.0}/pyproject.toml +2 -2
  18. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/conftest.py +2 -2
  19. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_config.py +148 -0
  20. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_data_builders.py +10 -5
  21. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_evaluate_helpers.py +20 -6
  22. euler_eval-2.31.0/tests/test_mask_sky_pairing.py +197 -0
  23. euler_eval-2.31.0/tests/test_mask_sky_prediction.py +239 -0
  24. euler_eval-2.31.0/tests/test_sky_iou.py +550 -0
  25. euler_eval-2.31.0/tests/test_sky_iou_cli.py +497 -0
  26. {euler_eval-2.30.0 → euler_eval-2.31.0}/LICENSE +0 -0
  27. {euler_eval-2.30.0 → euler_eval-2.31.0}/MANIFEST.in +0 -0
  28. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/README.md +0 -0
  29. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/distributions.md +0 -0
  30. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/feature-request-unified-evaluation-framework.md +0 -0
  31. {euler_eval-2.30.0 → euler_eval-2.31.0}/docs/validation.md +0 -0
  32. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/THIRD_PARTY_NOTICES.md +0 -0
  33. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/__init__.py +0 -0
  34. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/calibration.py +0 -0
  35. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/distribution_contract.py +0 -0
  36. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/distributions.py +0 -0
  37. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metric_sets.py +0 -0
  38. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/_natural_scene_models.py +0 -0
  39. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/absrel.py +0 -0
  40. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/daniel_error.py +0 -0
  41. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/depth_binned_error.py +0 -0
  42. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/depth_edge_f1.py +0 -0
  43. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/depth_standard.py +0 -0
  44. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/fade.py +0 -0
  45. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/fid_kid.py +0 -0
  46. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/gpu_depth_batch.py +0 -0
  47. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/gpu_image_batch.py +0 -0
  48. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/high_freq_energy.py +0 -0
  49. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/lpips_metric.py +0 -0
  50. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/niqe.py +0 -0
  51. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/normal_consistency.py +0 -0
  52. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/points3d_cloud.py +0 -0
  53. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/points3d_decomposition.py +0 -0
  54. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/points3d_distance.py +0 -0
  55. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/points3d_geometry.py +0 -0
  56. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/psnr.py +0 -0
  57. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rgb_edge_f1.py +0 -0
  58. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rgb_lpips.py +0 -0
  59. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rgb_psnr_ssim.py +0 -0
  60. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rho_a.py +0 -0
  61. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/rmse.py +0 -0
  62. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/scale_invariant_log.py +0 -0
  63. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/ssim.py +0 -0
  64. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/tail_errors.py +0 -0
  65. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/metrics/utils.py +0 -0
  66. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/sanity_checker.py +0 -0
  67. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/schemas/distribution-v1.schema.json +0 -0
  68. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/utils/hierarchy_parser.py +0 -0
  69. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval/validation.py +0 -0
  70. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/dependency_links.txt +0 -0
  71. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/entry_points.txt +0 -0
  72. {euler_eval-2.30.0 → euler_eval-2.31.0}/euler_eval.egg-info/top_level.txt +0 -0
  73. {euler_eval-2.30.0 → euler_eval-2.31.0}/init_cache.py +0 -0
  74. {euler_eval-2.30.0 → euler_eval-2.31.0}/scripts/generate_distribution_fixtures.py +0 -0
  75. {euler_eval-2.30.0 → euler_eval-2.31.0}/setup.cfg +0 -0
  76. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/__init__.py +0 -0
  77. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/distribution_contract_utils.py +0 -0
  78. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/README.md +0 -0
  79. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/counts-depth.eval.json +0 -0
  80. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/counts-points3d.eval.json +0 -0
  81. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/counts-sparsedepth.eval.json +0 -0
  82. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/future-sum-by-depth.eval.json +0 -0
  83. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/histogram-only.eval.json +0 -0
  84. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/per-file-only.eval.json +0 -0
  85. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/fixtures/distributions/scalar-only.eval.json +0 -0
  86. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/mock_files/calibration_output.json +0 -0
  87. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/mock_files/depth_output.json +0 -0
  88. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/mock_files/rgb_output.json +0 -0
  89. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/mock_files/segmentation_output.json +0 -0
  90. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_alignment.py +0 -0
  91. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_cli_device.py +0 -0
  92. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_cli_sparse_outputs.py +0 -0
  93. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_data.py +0 -0
  94. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_depth_alignment_output.py +0 -0
  95. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_depth_standard.py +0 -0
  96. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_distribution_contract.py +0 -0
  97. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_distributions.py +0 -0
  98. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_domain_metrics.py +0 -0
  99. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_init_cache.py +0 -0
  100. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_integration.py +0 -0
  101. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_meta_output.py +0 -0
  102. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_metrics_utils.py +0 -0
  103. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_normal_consistency.py +0 -0
  104. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_points_3d.py +0 -0
  105. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_points_3d_sparse.py +0 -0
  106. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_rgb_fid_output.py +0 -0
  107. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_rho_a.py +0 -0
  108. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_save_results.py +0 -0
  109. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_sparse_depth.py +0 -0
  110. {euler_eval-2.30.0 → euler_eval-2.31.0}/tests/test_validation.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: euler-eval
3
- Version: 2.30.0
3
+ Version: 2.31.0
4
4
  Summary: Evaluation toolkit for depth, RGB, ray and 3D point-map predictions
5
5
  Author-email: Daniel Rothenpieler <rothenpielerdaniel@gmail.com>
6
6
  License: MIT
@@ -33,7 +33,7 @@ Requires-Dist: opencv-python-headless>=4.5.0
33
33
  Requires-Dist: lpips>=0.1.4
34
34
  Requires-Dist: torchmetrics>=1.0
35
35
  Requires-Dist: tqdm>=4.62.0
36
- Requires-Dist: euler-loading>=2.15
36
+ Requires-Dist: euler-loading>=2.29
37
37
  Requires-Dist: ds-crawler>=2.10
38
38
  Requires-Dist: euler-metric-naming>=0.2
39
39
  Provides-Extra: logging
@@ -176,7 +176,8 @@ metric is written under, is in [Metrics](docs/metrics.md).
176
176
  | **Metadata-driven decoding** | Radial vs planar depth, value ranges, point-cloud column layout and loader choice all come from the dataset index — no per-dataset flags. |
177
177
  | **Honest spaces** | Relative models are scored `native` *and* `metric` (after scale/shift or a similarity gauge), so a scale failure never reads as a geometry failure. |
178
178
  | **Benchmark depth bins** | `--benchmark-depth-range MIN MAX` adds near/mid/far bins in square-root depth space, additive to the regular metrics. |
179
- | **Sky masking** | `--mask-sky` drops sky pixels using GT segmentation — from the metrics *and* from the alignment fit. |
179
+ | **Sky masking** | `--mask-sky` drops sky pixels using GT segmentation — from the metrics *and* from the alignment fit. `--mask-sky-prediction` does the same with each prediction's own `sky_mask`. |
180
+ | **Sky IoU** | `--sky-iou` scores predicted sky — a `sky_mask`, or depth reaching a fixed cap — against GT segmentation, without changing any depth metric. |
180
181
  | **Per-file + aggregate** | Dataset-level numbers plus per-image metrics in dataset-hierarchy order, in the same file. |
181
182
  | **Sanity checks** | Results are validated against configurable thresholds; implausible ranges, degenerate inputs and scale mismatches are reported instead of quietly scored. |
182
183
  | **Structured metric names** | Each result carries a `metricSet` envelope declaring its namespace, axes, units and metric directions. |
@@ -126,7 +126,8 @@ metric is written under, is in [Metrics](docs/metrics.md).
126
126
  | **Metadata-driven decoding** | Radial vs planar depth, value ranges, point-cloud column layout and loader choice all come from the dataset index — no per-dataset flags. |
127
127
  | **Honest spaces** | Relative models are scored `native` *and* `metric` (after scale/shift or a similarity gauge), so a scale failure never reads as a geometry failure. |
128
128
  | **Benchmark depth bins** | `--benchmark-depth-range MIN MAX` adds near/mid/far bins in square-root depth space, additive to the regular metrics. |
129
- | **Sky masking** | `--mask-sky` drops sky pixels using GT segmentation — from the metrics *and* from the alignment fit. |
129
+ | **Sky masking** | `--mask-sky` drops sky pixels using GT segmentation — from the metrics *and* from the alignment fit. `--mask-sky-prediction` does the same with each prediction's own `sky_mask`. |
130
+ | **Sky IoU** | `--sky-iou` scores predicted sky — a `sky_mask`, or depth reaching a fixed cap — against GT segmentation, without changing any depth metric. |
130
131
  | **Per-file + aggregate** | Dataset-level numbers plus per-image metrics in dataset-hierarchy order, in the same file. |
131
132
  | **Sanity checks** | Results are validated against configurable thresholds; implausible ranges, degenerate inputs and scale mismatches are reported instead of quietly scored. |
132
133
  | **Structured metric names** | Each result carries a `metricSet` envelope declaring its namespace, axes, units and metric directions. |
@@ -32,7 +32,8 @@ map `d_aligned = s · d_pred + t` by least squares over valid pixels.
32
32
 
33
33
  The fit uses pixels where GT and prediction are finite and GT > 0. With
34
34
  `--mask-sky`, sky pixels are excluded and the fit is trimmed to `gt ≤ P95(gt)`
35
- so residual sky outliers do not dominate.
35
+ so residual sky outliers do not dominate. Pixels in a prediction's `sky_mask`
36
+ are excluded from the fit with `--sky-depth` or `--mask-sky-prediction`.
36
37
 
37
38
  Declaring the prediction with the `relative_depth` or `affine_depth` config key
38
39
  (instead of `depth`) tells the evaluator the model is not metric, so
@@ -23,7 +23,9 @@ validation exits with status `1` before any dataset is opened.
23
23
  | `--skip-rays` | flag | off | Skip rays (spherical direction map) evaluation |
24
24
  | `--skip-points-3d` | flag | off | Skip points_3d (per-pixel 3D point map) evaluation |
25
25
  | `--mask-sky` | flag | off | Mask sky regions from metrics using GT segmentation |
26
+ | `--mask-sky-prediction` | flag | off | Exclude pixels in a prediction's `datasets[].sky_mask` from dense and sparse depth metrics and the alignment fit; predictions without a `sky_mask` are scored in full ([details](configuration.md#masking-predicted-sky)) |
26
27
  | `--sky-depth` | `float` | none | Cap GT and predicted depths in meters and fill predicted `sky_mask` pixels with this value; must be finite and positive |
28
+ | `--sky-iou` | flag | off | Score predicted sky against GT segmentation sky: IoU, precision and recall for the `explicit` (`datasets[].sky_mask`) and `implicit` (`depth >= --sky-depth`, metric predictions only) variants. Requires `gt.segmentation`; depth metrics are unchanged ([details](metrics.md#sky-segmentation)) |
27
29
  | `--no-sanity-check` | flag | off | Disable sanity checking of metric configurations |
28
30
  | `--metrics-config` | `str` | auto-detect | Path to `metrics_config.json` for sanity checking |
29
31
  | `--depth-alignment` | `{none,auto_affine,affine}` | `auto_affine` | Depth calibration mode ([details](alignment.md#depth-alignment)) |
@@ -54,9 +56,16 @@ euler-eval config.json --batch-size 32
54
56
  # Sky masking (requires gt.segmentation)
55
57
  euler-eval config.json --mask-sky -v
56
58
 
59
+ # Exclude each prediction's own sky (datasets[].sky_mask) from depth metrics
60
+ euler-eval config.json --mask-sky-prediction
61
+
57
62
  # Cap depth at 100 m and fill predicted sky at 100 m (optional datasets[].sky_mask)
58
63
  euler-eval config.json --sky-depth 100
59
64
 
65
+ # Sky IoU: explicit from datasets[].sky_mask, implicit at depth >= 100 m
66
+ # (requires gt.segmentation; the threshold is also the depth cap)
67
+ euler-eval config.json --sky-iou --sky-depth 100
68
+
60
69
  # Only depth evaluation
61
70
  euler-eval config.json --skip-rgb --skip-rays --skip-points-3d
62
71
 
@@ -109,6 +118,13 @@ per prediction dataset and modality with the GT and prediction paths, the
109
118
  resolved modality metadata (such as `radial_depth`), and the number of matched
110
119
  pairs.
111
120
 
121
+ With `--sky-iou`, each depth block ends with one line holding the pooled
122
+ (`pixel_pool`) sky IoU, precision and recall of every computed variant, for
123
+ example `Sky IoU, pixel_pool, 'model': explicit 0.9120 (precision 0.9500,
124
+ recall 0.9580); implicit (depth >= 100 m) 0.4410 (precision 0.9800, recall
125
+ 0.4450)`. Skipped variants, GT segmentation without sky and an empty implicit
126
+ sky are reported as warnings above it.
127
+
112
128
  A matched-pair count that is much lower than expected is the first sign that GT
113
129
  and predictions are not indexed consistently — see
114
130
  [Loader resolution & dataset metadata](configuration.md#loader-resolution--dataset-metadata).
@@ -71,7 +71,8 @@ required.
71
71
  | `gt.sparse_depth.path` | no\* | Sparse pointcloud GT, `(N,C)` points whose first columns are `x,y,z` in metres |
72
72
  | `gt.rays.path` | no\* | GT ray direction map dataset |
73
73
  | `gt.points_3d.path` | no\* | GT per-pixel 3D point map `(H,W,3)` in camera-frame metres |
74
- | `gt.segmentation.path` | no | GT segmentation (needed for `--mask-sky`) |
74
+ | `gt.segmentation.path` | no | GT segmentation (needed for `--mask-sky` and `--sky-iou`); see [GT segmentation](#gt-segmentation) |
75
+ | `gt.semantic_segmentation.path` | no | Alias of `gt.segmentation`; configure only one of the two |
75
76
  | `gt.calibration.path` | no | Calibration data (camera intrinsics); enables true camera-frame [surface normals](metrics.md#surface-normals-and-calibration) |
76
77
  | `gt.intrinsics.path` | with `gt.sparse_depth` | Camera intrinsics for pointcloud projection |
77
78
  | `gt.camera_extrinsics.path` | with `gt.sparse_depth` | Source-to-camera extrinsics (e.g. MUSES `lidar2rgb`) |
@@ -80,6 +81,35 @@ required.
80
81
 
81
82
  \* At least one of the five modality entries must be present.
82
83
 
84
+ ### GT segmentation
85
+
86
+ `gt.segmentation` is the canonical key. `gt.semantic_segmentation` is accepted
87
+ as an alias, for archives whose segmentation scope carries that name (MUSES,
88
+ for example); configuring both keys is an error. The segmentation is read only
89
+ when `--mask-sky` or `--sky-iou` needs it. `--mask-sky` without it warns and
90
+ disables itself, while `--sky-iou` without it exits with an error.
91
+
92
+ The dataset head's loader module must expose a `sky_mask` function, and the
93
+ head must declare the sky label as `skyclass` (euler-loading 2.29 or later).
94
+ euler-eval contains no label IDs.
95
+
96
+ Segmentation is per frame, so it is paired with the evaluated frames by
97
+ hierarchy path and file ID, like depth: its index must use the same folder
98
+ structure and IDs as the frames it masks. A frame without its own segmentation
99
+ file is not evaluated, and the run prints how many pairs the segmentation
100
+ dropped. When segmentation is read only for `--sky-iou`, it must not change
101
+ the evaluated pairs, so a segmentation that lacks frames is not loaded; the run
102
+ warns and the sky IoU is skipped. A segmentation whose IDs match no frame
103
+ stops the run with euler-loading's "No common IDs" error and per-modality file
104
+ counts.
105
+
106
+ The config key is also the default metadata scope. If the path has no
107
+ `#scope=` selector, no root-level metadata, and no scope of that name, the
108
+ scope is chosen among the archive's scopes: the one whose name or
109
+ `dataset-head.json` modality key is `segmentation` or `semantic_segmentation`
110
+ (both names are candidates under either key), or the only scope. If several
111
+ scopes match, or none among several, the run stops and asks for `#scope=`.
112
+
83
113
  ## `datasets` section
84
114
 
85
115
  Each entry needs a `name` and at least one prediction modality. Use only **one**
@@ -94,7 +124,7 @@ would be ambiguous.
94
124
  | `depth.path` | no\* | Predicted dense metric depth; also used against sparse pointcloud GT |
95
125
  | `relative_depth.path` | no\* | Predicted dense relative depth (scale/shift alignment supported) |
96
126
  | `affine_depth.path` | no\* | Predicted dense affine depth (scale/shift alignment supported) |
97
- | `sky_mask.path` | no | Auxiliary predicted sky mask for a dense depth entry; used with `--sky-depth` |
127
+ | `sky_mask.path` | no | Auxiliary predicted sky mask for a dense depth entry; used with `--sky-depth`, `--mask-sky-prediction` and `--sky-iou` |
98
128
  | `rays.path` | no\* | Predicted ray direction map dataset |
99
129
  | `points_3d.path` | no\* | Predicted per-pixel 3D point map `(H,W,3)` |
100
130
  | `output_file` | no | Custom results path (default: `eval.json` in the first modality path) |
@@ -123,8 +153,8 @@ The sky mask is paired with GT and predicted depth by sample ID and hierarchy.
123
153
  It supports the same archive, split, and `#scope=` selectors as depth, defaulting
124
154
  to the `sky_mask` metadata scope. Its loader must return a boolean mask with
125
155
  **True meaning sky**. For example, `generic_dense_depth.sky_mask` uses the RGB
126
- sky color in the mask dataset's `meta.sky_mask` (`[R, G, B]`) and returns a
127
- `(1, H, W)` tensor. Masks at another resolution follow the depth grid's crop
156
+ sky color in the mask dataset's `meta.skyclass` (`[R, G, B]`; the older
157
+ `meta.sky_mask` key is deprecated) and returns a `(1, H, W)` tensor. Masks at another resolution follow the depth grid's crop
128
158
  rule or use nearest-neighbor resizing.
129
159
 
130
160
  `--sky-depth` applies to dense and sparse depth evaluation. GT and both native
@@ -139,9 +169,37 @@ contributes only at projected points. Predicted sky stays in the scores where
139
169
  GT is valid; `--mask-sky` separately excludes pixels using GT segmentation and
140
170
  can be combined with `--sky-depth`.
141
171
 
142
- Without `--sky-depth`, the predicted mask is not loaded and depth evaluation
143
- uses the usual uncapped values. The effective cap and mask source are recorded
144
- in `eval.json` under `meta.eval_params.sky_depth` and `meta.pred.sky_mask`.
172
+ Without `--sky-depth`, `--mask-sky-prediction` or `--sky-iou`, the predicted
173
+ mask is not loaded and depth evaluation uses the usual uncapped values.
174
+ `--sky-iou` loads the mask
175
+ even without `--sky-depth`, but only to score it against GT sky
176
+ ([Sky segmentation](metrics.md#sky-segmentation)); the depth fill and the
177
+ affine-fit exclusion stay tied to `--sky-depth` (and `--mask-sky-prediction`),
178
+ so depth values do not change.
179
+ Because the mask is paired by sample ID, a mask that lacks some depth frames
180
+ also drops them from the depth metrics; the run prints how many. When the mask
181
+ is loaded only for `--sky-iou`, such a mask is not loaded at all: the run
182
+ warns, keeps every depth pair and skips the explicit variant. The effective cap and mask
183
+ source are recorded in `eval.json` under `meta.eval_params.sky_depth` and
184
+ `meta.pred.sky_mask`.
185
+
186
+ ### Masking predicted sky
187
+
188
+ `--mask-sky-prediction` excludes the pixels of a prediction's `sky_mask` from
189
+ dense and sparse depth evaluation, as `--mask-sky` does for GT sky: from every
190
+ metric, benchmark bin and histogram, and from the scale/shift fit. For sparse
191
+ depth, lidar points that project into predicted sky are not scored. A dataset
192
+ without a `sky_mask` is scored in full and the run warns. The flag loads the
193
+ mask without `--sky-depth`, combines with `--mask-sky` (the union of both skies
194
+ is excluded) and with `--sky-depth` (the cap still applies to the remaining
195
+ pixels). It does not change `--sky-iou`, which scores the masks themselves.
196
+
197
+ The excluded pixels come from the evaluated model, so two models are scored on
198
+ different pixel sets, and a model can lower its error by calling hard pixels
199
+ sky. Each result records how much it excluded under `meta.pred_sky_masking`
200
+ ([Results & output](output.md#predicted-sky-masking)); compare models with
201
+ this flag only alongside that fraction, or prefer `--mask-sky`, whose GT sky is
202
+ the same for every model.
145
203
 
146
204
  ## Sparse depth (pointcloud GT)
147
205
 
@@ -162,8 +220,9 @@ depth and `points_3d`, each prediction modality gets its own `eval.json` at its
162
220
  own path.
163
221
 
164
222
  Sparse depth does not require segmentation GT. `gt.segmentation` is loaded only
165
- when `--mask-sky` is set, and then excludes sky pixels from projected-point
166
- metrics and from scale/shift fitting.
223
+ when `--mask-sky` or `--sky-iou` is set. `--mask-sky` excludes sky pixels from
224
+ projected-point metrics and from scale/shift fitting; `--sky-iou` scores the
225
+ dense prediction's sky against it without changing any depth metric.
167
226
 
168
227
  ```json
169
228
  {
@@ -239,11 +298,15 @@ The scope defaults to the configuration key name, so the common case is compact:
239
298
 
240
299
  Use `#scope=<scope>` when an artifact scope is not named exactly like the config
241
300
  key, e.g. `"sparse_depth": { "path": "/data/capture.zip:test#scope=lidar" }`.
242
-
243
- Regular modalities (`rgb`, `depth`, `rays`, `points_3d`, `sparse_depth`)
244
- participate in file-ID intersection; calibration and pose data (`intrinsics`,
245
- `calibration`, `camera_extrinsics`, `lidar_extrinsics`, segmentation) are loaded
246
- hierarchically and do not.
301
+ GT segmentation is the exception: under either of its keys it also finds a
302
+ scope named after the other key, as described in [GT segmentation](#gt-segmentation).
303
+
304
+ Regular modalities (`rgb`, `depth`, `rays`, `points_3d`, `sparse_depth`, GT
305
+ segmentation and a predicted `sky_mask`) participate in file-ID intersection;
306
+ calibration and pose data (`intrinsics`, `calibration`, `camera_extrinsics`,
307
+ `lidar_extrinsics`) are loaded hierarchically and do not. A hierarchical file
308
+ applies to every frame at or below its folder, which suits one calibration per
309
+ scene but not per-frame data.
247
310
 
248
311
  ## `metrics_config.json`
249
312
 
@@ -76,6 +76,87 @@ the 45° sanity threshold.
76
76
  > plane read as nearly fronto-parallel and the focal length was unused. Values
77
77
  > are not comparable with earlier releases.
78
78
 
79
+ ### Sky segmentation
80
+
81
+ `--sky-iou` scores how well each prediction's sky matches the GT segmentation
82
+ sky. It is opt-in and read-only: no depth value, mask or alignment fit changes,
83
+ so every other key keeps its value. It requires `gt.segmentation` (or its alias
84
+ `gt.semantic_segmentation`); the sky label comes from that dataset head's
85
+ `skyclass`, as for `--mask-sky`.
86
+
87
+ There are two variants. They answer different questions, so they are reported
88
+ under separate keys and never merged:
89
+
90
+ | Variant | Predicted sky `P` | Computed when |
91
+ |---|---|---|
92
+ | `explicit` | The prediction's `datasets[].sky_mask` (True = sky) | The prediction has a `sky_mask` |
93
+ | `implicit` | `depth >= --sky-depth` | `--sky-depth` is set and the prediction is metric |
94
+
95
+ The implicit test reads the prediction after planar-to-radial conversion and
96
+ before calibration and capping. `+inf` counts as sky; NaN and values `<= 0` do
97
+ not. It is not computed for `relative_depth`, `affine_depth` or predictions
98
+ detected as normalized, since their values are not in meters. When both sources
99
+ exist, both variants are reported side by side; neither replaces the other. A
100
+ dataset with neither prints a warning and writes no sky keys.
101
+
102
+ | Metric | Key |
103
+ |---|---|
104
+ | Sky IoU / precision / recall | `depth.eval.sky_segmentation.{explicit,implicit}.{pixel_pool,image_mean,image_median}.{iou,precision,recall}` |
105
+ | Sparse-depth runs | `sparsedepth.eval.sky_segmentation.{explicit,implicit}.{pixel_pool,image_mean,image_median}.{iou,precision,recall}` |
106
+ | Per file | `{depth,sparsedepth}.eval.sky_segmentation.{explicit,implicit}.{iou,precision,recall}` |
107
+
108
+ `sky_segmentation` sits beside `native` and `metric`, not under them, because
109
+ the IoU does not depend on the evaluated depth space. Sparse-depth runs score
110
+ the dense prediction grid against the GT segmentation exactly like dense runs;
111
+ the sparse GT depth plays no part. Provenance, including the threshold and the
112
+ GT and predicted sky fractions, is recorded under
113
+ [`meta.sky_segmentation`](output.md#sky-segmentation-provenance).
114
+
115
+ If the GT segmentation lacks frames and `--mask-sky` is off, it is not loaded
116
+ and no sky IoU is computed, so that the depth pairs stay the same; see
117
+ [GT segmentation](configuration.md#gt-segmentation).
118
+
119
+ **Definitions.** GT sky `G` is the segmentation of the sample's own frame,
120
+ resized to the prediction grid with the same rule as GT depth (crop for the
121
+ VAE multiple-of-8 pattern, nearest neighbour otherwise). An explicit mask is
122
+ resized the same way. The domain `D` is the full prediction grid. Per sample:
123
+
124
+ ```text
125
+ I = |P ∩ G ∩ D| U = |(P ∪ G) ∩ D| T_p = |P ∩ D| T_g = |G ∩ D|
126
+ iou = I / U precision = I / T_p recall = I / T_g
127
+ ```
128
+
129
+ Each value is undefined when its denominator is 0. `image_mean` and
130
+ `image_median` reduce the defined per-image values. `pixel_pool` is
131
+ `ΣI / ΣU`, `ΣI / ΣT_p` and `ΣI / ΣT_g` over all samples, so a sample with an
132
+ undefined IoU (no sky in `P` or `G`) is left out of the image reducers but
133
+ still adds its zero counts to the pooled sums. Undefined dataset values are
134
+ omitted from `eval.json`.
135
+
136
+ **Interpreting the numbers.**
137
+
138
+ - **Explicit IoU** measures agreement between a dedicated sky prediction and GT
139
+ sky. It does not depend on depth quality.
140
+ - **Implicit IoU** measures whether the depth output reaches `--sky-depth` on
141
+ GT sky pixels. It does not measure whether the regressor recognizes sky. A
142
+ regressor that outputs about 95 m over the sky, as UniDepth V2 does in fog,
143
+ never reaches a 250 m cap; its implicit IoU is close to zero by construction,
144
+ even if it places the sky correctly relative to everything else. The same
145
+ model at `--sky-depth 80` can score high. Implicit IoU therefore depends on
146
+ both the model's saturation behaviour and the cap.
147
+
148
+ Consequences:
149
+
150
+ - Never present implicit and explicit IoU as the same quantity, and always
151
+ state the threshold with an implicit value (`meta.sky_segmentation.threshold_m`).
152
+ - Do not choose `--sky-depth` on the benchmark to maximize implicit IoU. That
153
+ is the same leakage as choosing the fill constant on the test set. Fix the
154
+ threshold beforehand, or report several runs at fixed thresholds.
155
+ - Pooled IoU weights images by their sky area, so a few large-sky images can
156
+ dominate it. `image_median` shows typical behaviour.
157
+ - Precision and recall separate the two failure modes. Low recall with high
158
+ precision usually means the regressor does not saturate at the cap.
159
+
79
160
  ## Sparse depth
80
161
 
81
162
  With a sparse pointcloud GT, only metrics that stay meaningful at isolated
@@ -86,6 +167,7 @@ skipped. Results serialize under the `sparsedepth.eval` root:
86
167
  |---|---|
87
168
  | Standard | `sparsedepth.eval.{native,metric}.standard.{image_mean,image_median,pixel_pool}.*` |
88
169
  | AbsRel / RMSE / SILog | `sparsedepth.eval.{native,metric}.depth_metrics.*` |
170
+ | Sky segmentation (`--sky-iou`) | `sparsedepth.eval.sky_segmentation.*` ([details](#sky-segmentation)) |
89
171
 
90
172
  ## RGB
91
173
 
@@ -78,6 +78,65 @@ GT/prediction resolution mismatch was resolved — `none` when they already matc
78
78
  `vae_crop` for the multiple-of-8 crop that latent models produce, `resize`
79
79
  otherwise, and `pointcloud_projection` when GT was a sparse cloud.
80
80
 
81
+ ### Sky segmentation provenance
82
+
83
+ With `--sky-iou`, dense and sparse depth results record
84
+ `meta.eval_params.sky_iou: true` and a `meta.sky_segmentation` block. Without
85
+ the flag, neither field exists. Values below are illustrative:
86
+
87
+ ```json
88
+ "sky_segmentation": {
89
+ "explicit": {
90
+ "available": true, "source": "pred.sky_mask", "samples": 500,
91
+ "samples_undefined_iou": 3, "pred_sky_fraction": 0.19
92
+ },
93
+ "implicit": { "available": false, "reason": "prediction is not metric (relative)" },
94
+ "gt_source": "gt.segmentation",
95
+ "domain": "prediction_grid",
96
+ "samples": 500,
97
+ "samples_without_gt": 0,
98
+ "gt_sky_fraction": 0.21
99
+ }
100
+ ```
101
+
102
+ | Field | Meaning |
103
+ |---|---|
104
+ | `explicit`, `implicit` | Whether each variant was computed. A computed variant gives its `source`, how many `samples` it scored, how many had an undefined IoU (no sky in prediction or GT), and its pooled predicted sky fraction; the implicit one adds `rule: "depth >= threshold_m"`. A skipped variant gives the `reason` instead |
105
+ | `gt_source` | The config key the GT segmentation came from |
106
+ | `threshold_m` | The `--sky-depth` threshold of the implicit variant; present only when that variant was computed |
107
+ | `domain` | Pixels scored: `prediction_grid`, the full prediction grid |
108
+ | `samples` | Depth pairs evaluated |
109
+ | `samples_without_gt` | Pairs left out because no GT segmentation of their own frame was found |
110
+ | `gt_sky_fraction` | Pooled GT sky fraction over the scored samples |
111
+
112
+ ### Predicted-sky masking
113
+
114
+ With `--mask-sky-prediction`, dense and sparse depth results record
115
+ `meta.eval_params.mask_sky_prediction: true` and how much the prediction's own
116
+ sky removed, since that differs from model to model:
117
+
118
+ ```json
119
+ "pred_sky_masking": {
120
+ "source": "pred.sky_mask",
121
+ "samples": 500,
122
+ "samples_with_sky_mask": 500,
123
+ "gt_valid_pixels": 1048576000,
124
+ "excluded_gt_pixels": 220200960,
125
+ "excluded_gt_fraction": 0.21
126
+ }
127
+ ```
128
+
129
+ `gt_valid_pixels` counts the GT pixels (projected lidar points for sparse
130
+ depth) that would be scored without the flag, after `--mask-sky` if set;
131
+ `excluded_gt_pixels` is how many of them fell in predicted sky.
132
+
133
+ When sky keys are written, the metric-set axes also declare them:
134
+ `sky_segmentation` joins the `category` values, an optional `variant` axis
135
+ (`explicit`, `implicit`) follows `category`, and `space` becomes optional
136
+ because these keys have no depth space. `metricDescriptions` gains `iou`,
137
+ `precision` and `recall` (higher is better, bounds `[0, 1]`). Every other key
138
+ decomposes exactly as before.
139
+
81
140
  ## Spaces and canonical aliases
82
141
 
83
142
  Depth-like and points-3d results are nested by space: