euler-eval 2.33.0__tar.gz → 2.35.0__tar.gz

{euler_eval-2.33.0/euler_eval.egg-info → euler_eval-2.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-eval
-Version: 2.33.0
+Version: 2.35.0
 Summary: Evaluation toolkit for depth, RGB, ray and 3D point-map predictions
 Author-email: Daniel Rothenpieler <rothenpielerdaniel@gmail.com>
 License: MIT

{euler_eval-2.33.0 → euler_eval-2.35.0}/docs/configuration.md

@@ -272,7 +272,7 @@ command-line flags and under its own name:
 |---|---|---|
 | `name` | yes | Evaluation name: results are written as `<name>.json` in each prediction dataset's `.eval/` folder ([where results land](output.md#where-results-land)). Same rules as `--eval-name`; unique |
 | `args` | no | Command-line tokens this experiment adds, exactly as you would type them; numbers may be written as JSON numbers |
-| `modalities` | no | The configured modalities this experiment may load (default: all). Names: `depth` (also covers `relative_depth` and `affine_depth` predictions), `sparse_depth`, `rgb`, `rays`, `points_3d`, `sky_mask`, `segmentation` (also covers `semantic_segmentation`), `calibration`, `intrinsics`, `camera_extrinsics`, `lidar_extrinsics` |
+| `modalities` | no | The prediction modalities (`datasets[]` entries) this experiment may load (default: all): `depth` (also covers `relative_depth` and `affine_depth`), `rgb`, `rays`, `points_3d`, `sky_mask`. The `gt` section is always used as configured |
 
 - **The command line is the base.** Each experiment runs with the flags given on
   the command line followed by its own `args`, so for a flag that takes a
@@ -283,17 +283,20 @@ command-line flags and under its own name:
   validated before the first one runs. `--eval-name` and `--experiment` are not
   allowed in `args`, and `--eval-name` cannot be combined with an `experiments`
   section, because `name` names the results.
-- **Modalities.** An experiment with `modalities` runs as if the unlisted GT
-  inputs and prediction entries were not in the config; a name covers both
-  sides. A listed modality is still read only when the evaluation needs it,
-  e.g. `sky_mask` only with `--sky-depth`, `--mask-sky-prediction` or
-  `--sky-iou`. This is how to compare runs where a modality's presence changes
-  the output: above, `capped_no_mask` caps depth at 80 m without filling the
-  predicted sky. A dataset left without a prediction is not evaluated in that
-  experiment, and the reduced config must still be valid (e.g. `sparse_depth`
-  needs `intrinsics` and `camera_extrinsics`). Listing a modality the config
-  does not have only prints a warning, so one experiment list can serve several
-  generated configs.
+- **Modalities.** An experiment with `modalities` runs as if the unlisted
+  prediction entries were not in `datasets[]`; a `sky_mask` goes with its depth
+  prediction. GT inputs are not selected this way: they stay as configured,
+  and flags decide which ones an evaluation reads (`--mask-sky` the
+  segmentation, for example), so GT names such as `segmentation` or
+  `sparse_depth` are rejected. A listed modality is still read only when the
+  evaluation needs it, e.g. `sky_mask` only with `--sky-depth`,
+  `--mask-sky-prediction` or `--sky-iou`; with just `--mask-sky`, listing
+  `sky_mask` changes nothing. This is how to compare runs where a modality's
+  presence changes the output: above, `capped_no_mask` caps depth at 80 m
+  without filling the predicted sky. A dataset left without a prediction is
+  not evaluated in that experiment. Listing a modality no dataset has only
+  prints a warning, so one experiment list can serve several generated
+  configs.
 - **Selecting experiments.** By default all experiments run, in config order.
   `--experiment NAME` (repeatable) runs only the named ones, e.g. one per task
   of a job array.

{euler_eval-2.33.0 → euler_eval-2.35.0}/euler_eval/cli.py

@@ -273,24 +273,28 @@ def _points_3d_eval_axes(*, distributions: bool = False) -> dict[str, AxisDeclar
 
 _PRED_DEPTH_KEYS = ("depth", "relative_depth", "affine_depth")
 
-#: Names an experiment's ``modalities`` may list, and the GT and prediction
-#: config keys each one covers.
+#: Prediction modalities an experiment's ``modalities`` may list, and the
+#: ``datasets[]`` keys each one covers. GT inputs are not selected this way:
+#: they are used as configured, and flags such as ``--mask-sky`` decide
+#: which of them an evaluation reads.
 _EXPERIMENT_MODALITIES = {
     "depth": _PRED_DEPTH_KEYS,
-    "sparse_depth": ("sparse_depth",),
     "rgb": ("rgb",),
     "rays": ("rays",),
     "points_3d": ("points_3d",),
     "sky_mask": ("sky_mask",),
-    "segmentation": GT_SEGMENTATION_KEYS,
-    "calibration": ("calibration",),
-    "intrinsics": ("intrinsics",),
-    "camera_extrinsics": ("camera_extrinsics",),
-    "lidar_extrinsics": ("lidar_extrinsics",),
 }
 _MODALITY_OF_CONFIG_KEY = {
     key: name for name, keys in _EXPERIMENT_MODALITIES.items() for key in keys
 }
+_GT_ONLY_KEYS = (
+    "sparse_depth",
+    *GT_SEGMENTATION_KEYS,
+    "calibration",
+    "intrinsics",
+    "camera_extrinsics",
+    "lidar_extrinsics",
+)
 
 # Downstream eval consumers validate metricSet.metricNamespace with a stricter
 # first-segment rule than euler_metric_naming's modality validator. Keep the
@@ -1166,6 +1170,12 @@ def _validate_experiment_modalities(modalities, label: str) -> None:
             'e.g. ["depth", "sky_mask"]'
         )
     for name in modalities:
+        if name in _GT_ONLY_KEYS:
+            raise ValueError(
+                f"{label}.modalities: {name!r} is a GT input; 'modalities' selects "
+                "prediction modalities, and GT inputs are used as configured "
+                "(flags such as --mask-sky decide which ones are read)"
+            )
         if name not in _EXPERIMENT_MODALITIES:
             covered_by = _MODALITY_OF_CONFIG_KEY.get(name)
             hint = f" (use {covered_by!r}, which covers it)" if covered_by else ""
@@ -1185,21 +1195,21 @@ def _has_prediction(entry: dict) -> bool:
 
 
 def _configured_modalities(config: dict) -> set[str]:
-    sections = [config["gt"], *config["datasets"]]
     return {
         _MODALITY_OF_CONFIG_KEY[key]
-        for section in sections
-        for key in section
+        for entry in config["datasets"]
+        for key in entry
         if key in _MODALITY_OF_CONFIG_KEY
     }
 
 
 def _experiment_config(config: dict, modalities: list[str] | None) -> dict:
-    """Return *config* reduced to the modalities an experiment lists.
+    """Return *config* with only the prediction modalities an experiment lists.
 
-    Unlisted GT inputs and prediction entries are left out, as if they were
-    not configured. A dataset left without a prediction is not evaluated, and
-    a ``sky_mask`` is dropped with the depth prediction it belongs to.
+    Unlisted ``datasets[]`` entries are left out, as if they were not
+    configured; the ``gt`` section is unchanged. A dataset left without a
+    prediction is not evaluated, and a ``sky_mask`` is dropped with the depth
+    prediction it belongs to.
     """
     if modalities is None:
         return config
@@ -1220,7 +1230,7 @@ def _experiment_config(config: dict, modalities: list[str] | None) -> dict:
             entry.pop("sky_mask", None)
         if _has_prediction(entry):
             datasets.append(entry)
-    return {**config, "gt": reduced(config["gt"]), "datasets": datasets}
+    return {**config, "datasets": datasets}
 
 
 def load_config(config_path: str) -> dict:
@@ -1652,7 +1662,6 @@ def _experiment_runs(
         parser.raise_errors = True
         try:
             if modalities is not None:
-                validate_gt_config(run_config["gt"])
                 if not run_config["datasets"]:
                     raise ValueError(
                         "'modalities' leave no dataset with a prediction to evaluate"
@@ -1679,7 +1688,7 @@ def _experiment_runs(
             if run_args.sky_iou and _gt_segmentation_entry(run_config["gt"])[1] is None:
                 raise ValueError(
                     "--sky-iou requires gt.segmentation (or its alias "
-                    "gt.semantic_segmentation), which this experiment does not load"
+                    "gt.semantic_segmentation) in config"
                 )
             runs.append(
                 _prepare_run(run_args, run_config, name=name, extra_args=tuple(extra))

{euler_eval-2.33.0 → euler_eval-2.35.0/euler_eval.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-eval
-Version: 2.33.0
+Version: 2.35.0
 Summary: Evaluation toolkit for depth, RGB, ray and 3D point-map predictions
 Author-email: Daniel Rothenpieler <rothenpielerdaniel@gmail.com>
 License: MIT

{euler_eval-2.33.0 → euler_eval-2.35.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "euler-eval"
-version = "2.33.0"
+version = "2.35.0"
 description = "Evaluation toolkit for depth, RGB, ray and 3D point-map predictions"
 readme = "README.md"
 requires-python = ">=3.9"

{euler_eval-2.33.0 → euler_eval-2.35.0}/tests/test_experiments.py

@@ -1,12 +1,14 @@
 """Config ``experiments``: several named argument sets in one invocation."""
 
 import json
+import zipfile
 from pathlib import Path
 
 import pytest
 
 from euler_eval import cli
 from euler_eval.cli import default_eval_path, validate_experiments_config
+from tests.test_cli_sparse_outputs import _sparse_depth_results
 from tests.test_depth_alignment_output import _patch_depth_metrics
 from tests.test_sky_iou_cli import write_gt_segmentation
 
@@ -293,20 +295,20 @@ def test_modalities_decide_whether_the_sky_mask_is_used(
     assert capped["meta"]["pred"].get("sky_mask") is None
 
 
-def test_modalities_leave_out_gt_inputs(monkeypatch, tmp_path, indexed_sky_prediction):
+def test_modalities_keep_every_gt_input(monkeypatch, tmp_path, indexed_sky_prediction):
+    """'modalities' selects predictions; GT segmentation stays for --mask-sky."""
     _patch_depth_metrics(monkeypatch)
     pred = indexed_sky_prediction["pred"]
     seg = write_gt_segmentation(tmp_path / "segmentation")
     _main(monkeypatch, _sky_config(
         tmp_path, indexed_sky_prediction,
-        [{"name": "all"}, {"name": "no_seg", "modalities": ["depth", "sky_mask"]}],
+        [{"name": "eval", "modalities": ["depth"], "args": ["--mask-sky"]}],
         gt_extra={"segmentation": {"path": seg}},
     ))
 
-    full = _results(pred, "all")["protocol"]["parameters"]
-    reduced = _results(pred, "no_seg")["protocol"]["parameters"]
-    assert full["reference.segmentation.dataset"] == "sky_iou_test"
-    assert reduced["reference.segmentation.dataset"] is None
+    parameters = _results(pred, "eval")["protocol"]["parameters"]
+    assert parameters["population.gt_sky_masking"] is True
+    assert parameters["reference.segmentation.dataset"] == "sky_iou_test"
 
 
 def test_unconfigured_modalities_only_warn(
@@ -314,40 +316,46 @@ def test_unconfigured_modalities_only_warn(
 ):
     _patch_depth_metrics(monkeypatch)
     _main(monkeypatch, _sky_config(tmp_path, indexed_sky_prediction, [
-        {"name": "eval", "modalities": ["depth", "calibration"]},
+        {"name": "eval", "modalities": ["depth", "rays"]},
     ]))
 
-    assert "modalities ['calibration'] are not configured" in capsys.readouterr().err
+    assert "modalities ['rays'] are not configured" in capsys.readouterr().err
     assert _results(indexed_sky_prediction["pred"], "eval") is not None
 
 
 @pytest.mark.parametrize(
-    ("experiment", "message"),
+    ("experiment", "with_segmentation", "message"),
     [
         (
             {"name": "iou", "args": ["--sky-iou"], "modalities": ["depth"]},
+            False,
             "experiments[0] ('iou'): --sky-iou requires gt.segmentation",
         ),
         (
+            # A sky_mask is dropped with the depth prediction it belongs to.
             {"name": "masks", "modalities": ["sky_mask"]},
-            "experiments[0] ('masks'): gt must have at least one of",
+            True,
+            "experiments[0] ('masks'): 'modalities' leave no dataset with a prediction",
         ),
         (
             {"name": "rgb", "modalities": ["rgb"]},
+            True,
             "experiments[0] ('rgb'): 'modalities' leave no dataset with a prediction",
         ),
     ],
 )
 def test_modalities_are_checked_before_any_work(
-    monkeypatch, capsys, tmp_path, indexed_sky_prediction, experiment, message
+    monkeypatch, capsys, tmp_path, indexed_sky_prediction, experiment,
+    with_segmentation, message,
 ):
     monkeypatch.setattr(cli, "evaluate_depth_samples", pytest.fail)
-    gt_rgb = tmp_path / "gt_rgb"
-    gt_rgb.mkdir()
-    seg = write_gt_segmentation(tmp_path / "segmentation")
+    gt_extra = {}
+    if with_segmentation:
+        gt_extra["segmentation"] = {
+            "path": write_gt_segmentation(tmp_path / "segmentation")
+        }
     config_path = _sky_config(
-        tmp_path, indexed_sky_prediction, [experiment],
-        gt_extra={"rgb": {"path": str(gt_rgb)}, "segmentation": {"path": seg}},
+        tmp_path, indexed_sky_prediction, [experiment], gt_extra=gt_extra
     )
     with pytest.raises(SystemExit) as exc:
         _main(monkeypatch, config_path)
@@ -356,7 +364,76 @@ def test_modalities_are_checked_before_any_work(
     assert f"Error loading config: {message}" in capsys.readouterr().err
 
 
-def test_experiment_config_keeps_only_listed_modalities():
+def test_sparse_gt_config_with_prediction_modalities(monkeypatch, tmp_path):
+    """The reported MUSES config: experiments choose predictions, GT stays whole."""
+    paths = {}
+    for name in ("lidar", "camera", "semantic", "depth", "dehazed", "points_3d",
+                 "sky_mask"):
+        paths[name] = tmp_path / f"{name}.zip"
+        with zipfile.ZipFile(paths[name], "w") as zf:
+            zf.writestr("placeholder", b"data")
+    config_path = tmp_path / "config.json"
+    config_path.write_text(json.dumps({
+        "experiments": [
+            {"name": "eval", "modalities": ["depth"], "args": ["--mask-sky"]},
+            {"name": "gt_sky_with_mask", "modalities": ["depth", "sky_mask"],
+             "args": ["--mask-sky"]},
+            {"name": "gt_and_pred_sky", "modalities": ["depth", "sky_mask"],
+             "args": ["--mask-sky", "--mask-sky-prediction"]},
+        ],
+        "gt": {
+            "sparse_depth": {"path": str(paths["lidar"])},
+            "rgb": {"path": f"{paths['camera']}:fog_day"},
+            "intrinsics": {"path": f"{paths['camera']}#scope=intrinsics"},
+            "camera_extrinsics": {"path": f"{paths['camera']}#scope=camera_extrinsics"},
+            "segmentation": {"path": str(paths["semantic"])},
+        },
+        "datasets": [{
+            "name": "SD 2.1. Mari",
+            "depth": {"path": str(paths["depth"])},
+            "rgb": {"path": str(paths["dehazed"])},
+            "points_3d": {"path": str(paths["points_3d"])},
+            "sky_mask": {"path": str(paths["sky_mask"])},
+        }],
+        "distributions": {"n_bins": 30, "scale": "log", "min_error": 0.001,
+                          "max_error": 240.0},
+    }))
+
+    built, evaluated = [], []
+    monkeypatch.setattr(
+        cli, "build_sparse_depth_eval_dataset",
+        lambda **kwargs: built.append(kwargs) or [object()],
+    )
+    monkeypatch.setattr(
+        cli, "get_sparse_depth_metadata", lambda _ds: {"pred_radial_depth": False}
+    )
+    monkeypatch.setattr(
+        cli, "evaluate_sparse_depth_samples",
+        lambda **kwargs: evaluated.append(kwargs) or _sparse_depth_results(),
+    )
+    for unused in ("evaluate_rgb_samples", "evaluate_points_3d_sparse_samples"):
+        monkeypatch.setattr(cli, unused, pytest.fail)
+    _main(monkeypatch, config_path)
+
+    gt_masking = [kwargs["sky_mask_enabled"] for kwargs in evaluated]
+    pred_masking = [kwargs["pred_sky_mask_enabled"] for kwargs in evaluated]
+    assert gt_masking == [True, True, True]
+    assert pred_masking == [False, False, True]
+    assert [kwargs["segmentation_path"] for kwargs in built] == [str(paths["semantic"])] * 3
+    # The sky_mask is only read by a flag that uses it (here --mask-sky-prediction).
+    assert [kwargs["pred_sky_mask_path"] for kwargs in built] == [
+        None, None, str(paths["sky_mask"]),
+    ]
+    with zipfile.ZipFile(paths["depth"]) as zf:
+        saved = {
+            name: json.loads(zf.read(f".eval/{name}.json"))
+            for name in ("eval", "gt_sky_with_mask", "gt_and_pred_sky")
+        }
+    assert [s["protocol"]["parameters"]["method.pred_sky_mask"]
+            for s in saved.values()] == [False, False, True]
+
+
+def test_experiment_config_keeps_only_listed_predictions():
     config = {
         "gt": {
             "name": "GT",
@@ -375,17 +452,17 @@ def test_experiment_config_keeps_only_listed_modalities():
         "distributions": True,
     }
 
-    reduced = cli._experiment_config(config, ["depth", "segmentation"])
+    reduced = cli._experiment_config(config, ["depth"])
 
-    assert reduced["gt"] == {
-        "name": "GT",
-        "depth": {"path": "/gt/depth"},
-        "semantic_segmentation": {"path": "/gt/seg"},
-    }
+    assert reduced["gt"] == config["gt"]
     assert reduced["datasets"] == [
         {"name": "a", "output_file": "/out/a.json", "relative_depth": {"path": "/a/depth"}},
     ]
     assert reduced["distributions"] is True
+    assert cli._experiment_config(config, ["depth", "sky_mask"])["datasets"] == [{
+        "name": "a", "output_file": "/out/a.json",
+        "relative_depth": {"path": "/a/depth"}, "sky_mask": {"path": "/a/sky"},
+    }]
     assert cli._experiment_config(config, ["rgb", "sky_mask"])["datasets"] == [
         {"name": "a", "output_file": "/out/a.json", "rgb": {"path": "/a/rgb"}},
         {"name": "b", "rgb": {"path": "/b/rgb"}},
@@ -399,7 +476,10 @@ def test_experiment_config_keeps_only_listed_modalities():
         ([], "non-empty array of modality names"),
         ("depth", "non-empty array of modality names"),
         (["relative_depth"], r"unknown modality 'relative_depth' \(use 'depth'"),
-        (["semantic_segmentation"], r"\(use 'segmentation', which covers it\)"),
+        (["segmentation"], "'segmentation' is a GT input"),
+        (["semantic_segmentation"], "'semantic_segmentation' is a GT input"),
+        (["sparse_depth"], "'sparse_depth' is a GT input"),
+        (["intrinsics"], "'intrinsics' is a GT input"),
         (["normals"], "unknown modality 'normals'"),
         (["depth", "depth"], r"lists \['depth'\] more than once"),
     ],