euler-preprocess 3.16.0__tar.gz → 3.17.0__tar.gz

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: euler-preprocess
-Version: 3.16.0
+Version: 3.17.0
 Summary: Physics-based preprocessing (fog, etc.) for RGB+depth datasets
 Author-email: Daniel Rothenpieler <rothenpielerdaniel@gmail.com>
 License-Expression: MIT
@@ -9,7 +9,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: Pillow
-Requires-Dist: euler-loading>=2.24.0
+Requires-Dist: euler-loading>=2.29.0
 Requires-Dist: ds-crawler>=2.11.0
 Requires-Dist: euler-dataset-contract>=0.5.0
 Provides-Extra: gpu
@@ -119,6 +119,15 @@ loader.
 | `sky-depth` | `depth`, `semantic_segmentation` | — |
 | `radial` | `depth`, `intrinsics` | — |
 
+`fog` and `sky-depth` need `semantic_segmentation` as a boolean sky mask. They load
+it with the `sky_mask` function of the euler-loading module that the index's
+dataset head names, so a `class_segmentation` index works as is: the sky is the
+head's `skyclass` (Synscapes `[23, 23, 23]`, Real Drive Sim `[29, 0, 0]`, VKITTI2
+`[90, 200, 255]`), and the log names the loader and the sky definition in use.
+Set `"function"` in the modality entry to use another loader function, or
+`"function": null` for the one the head declares. A non-binary mask raises an
+error rather than marking every non-zero class as sky.
+
 **Per-sample vs. hierarchical modalities.** You don't declare a modality's role:
 it is inferred from the ds-crawler indexes. A modality whose qualified IDs
 (hierarchy path plus file ID) match the transform's per-sample inputs is joined per
@@ -667,7 +676,8 @@ where sky depth is encoded as zero or infinity and needs a large finite value.
 }
 ```
 
-`sky_depth_value` defaults to `1000.0`.
+`sky_depth_value` defaults to `1000.0`. The sky mask comes from the dataset head as
+described under [Dataset Config](#dataset-config).
 
 CLI runs write a source-backed depth dataset mirroring the input depth modality's paths,
 filenames, extensions, and metadata. Standalone `SkyDepthTransform(...)` usage keeps the

euler_preprocess-3.16.0/euler_preprocess.egg-info/PKG-INFO → euler_preprocess-3.17.0/README.md

@@ -1,23 +1,3 @@
-Metadata-Version: 2.4
-Name: euler-preprocess
-Version: 3.16.0
-Summary: Physics-based preprocessing (fog, etc.) for RGB+depth datasets
-Author-email: Daniel Rothenpieler <rothenpielerdaniel@gmail.com>
-License-Expression: MIT
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: numpy
-Requires-Dist: Pillow
-Requires-Dist: euler-loading>=2.24.0
-Requires-Dist: ds-crawler>=2.11.0
-Requires-Dist: euler-dataset-contract>=0.5.0
-Provides-Extra: gpu
-Requires-Dist: torch; extra == "gpu"
-Provides-Extra: progress
-Requires-Dist: tqdm; extra == "progress"
-Dynamic: license-file
-
 <!-- euler header — shared across the euler packages.
      Per package, change only: the <h1>, the tagline, and the badge URLs. -->
 <p align="center">
@@ -119,6 +99,15 @@ loader.
 | `sky-depth` | `depth`, `semantic_segmentation` | — |
 | `radial` | `depth`, `intrinsics` | — |
 
+`fog` and `sky-depth` need `semantic_segmentation` as a boolean sky mask. They load
+it with the `sky_mask` function of the euler-loading module that the index's
+dataset head names, so a `class_segmentation` index works as is: the sky is the
+head's `skyclass` (Synscapes `[23, 23, 23]`, Real Drive Sim `[29, 0, 0]`, VKITTI2
+`[90, 200, 255]`), and the log names the loader and the sky definition in use.
+Set `"function"` in the modality entry to use another loader function, or
+`"function": null` for the one the head declares. A non-binary mask raises an
+error rather than marking every non-zero class as sky.
+
 **Per-sample vs. hierarchical modalities.** You don't declare a modality's role:
 it is inferred from the ds-crawler indexes. A modality whose qualified IDs
 (hierarchy path plus file ID) match the transform's per-sample inputs is joined per
@@ -667,7 +656,8 @@ where sky depth is encoded as zero or infinity and needs a large finite value.
 }
 ```
 
-`sky_depth_value` defaults to `1000.0`.
+`sky_depth_value` defaults to `1000.0`. The sky mask comes from the dataset head as
+described under [Dataset Config](#dataset-config).
 
 CLI runs write a source-backed depth dataset mirroring the input depth modality's paths,
 filenames, extensions, and metadata. Standalone `SkyDepthTransform(...)` usage keeps the

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/cli.py

@@ -178,7 +178,14 @@ def _run_transform(args: argparse.Namespace, transform_class: type) -> int:
 
     required_modalities = transform_class.REQUIRED_MODALITIES
     required_hierarchical = transform_class.REQUIRED_HIERARCHICAL_MODALITIES or None
-    dataset = build_dataset(config, required_modalities, required_hierarchical)
+    # euler-loading logs which loader (and sky definition) each modality uses.
+    get_logger("euler_loading")
+    dataset = build_dataset(
+        config,
+        required_modalities,
+        required_hierarchical,
+        transform_class.MODALITY_FUNCTIONS,
+    )
     output_backends = prepare_output_backends(config, dataset, transform_class)
     primary_backend = next(iter(output_backends.values()))
     samples = _select_configured_samples(config, dataset, logger)

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/common/dataset.py

@@ -5,26 +5,27 @@ from collections import Counter
 from euler_preprocess.common.logging import get_logger
 
 
-def _make_modality(entry: str | dict):
+def _make_modality(entry: str | dict, function: str | None = None):
+    """Build a ``Modality``; an entry's own ``function`` replaces *function*."""
     from euler_loading import Modality
 
     if isinstance(entry, str):
-        return Modality(entry)
-    return Modality(
-        entry["path"],
-        **{
-            key: entry[key]
-            for key in (
-                "split",
-                "metadata_scope",
-                "collapse_single",
-                "cache",
-                "applies_to",
-                "hierarchy_scope",
-            )
-            if key in entry
-        },
-    )
+        return Modality(entry, function=function)
+    options = {
+        key: entry[key]
+        for key in (
+            "split",
+            "metadata_scope",
+            "collapse_single",
+            "cache",
+            "applies_to",
+            "hierarchy_scope",
+            "function",
+        )
+        if key in entry
+    }
+    options.setdefault("function", function)
+    return Modality(entry["path"], **options)
 
 
 def _load_index_tree(name: str, modality) -> dict:
@@ -151,6 +152,7 @@ def build_dataset(
     config: dict,
     required_modalities: set[str],
     required_hierarchical: set[str] | None = None,
+    modality_functions: dict[str, str] | None = None,
 ):
     """Build a ``MultiModalDataset`` from a config dict.
 
@@ -169,6 +171,12 @@ def build_dataset(
     same way.  *required_modalities* are per-sample inputs and anchor that
     inference; *required_hierarchical* names further required modalities,
     whose role is inferred like any other.
+
+    *modality_functions* maps a modality to the loader function to use from
+    the module its dataset head names (a transform's ``MODALITY_FUNCTIONS``),
+    e.g. ``sky_mask`` for a ``class_segmentation`` index; the sky then comes
+    from the head's ``skyclass``.  A modality entry's own ``function`` key
+    overrides it, and ``"function": null`` keeps the head's function.
     """
     from euler_loading import MultiModalDataset, SamplePreprocessor
 
@@ -191,7 +199,11 @@ def build_dataset(
             f"'modalities' must contain at least: {', '.join(sorted(required))}"
         )
 
-    modalities = {name: _make_modality(entry) for name, entry in configured.items()}
+    functions = modality_functions or {}
+    modalities = {
+        name: _make_modality(entry, functions.get(name))
+        for name, entry in configured.items()
+    }
     regular, hierarchical = _resolve_roles(modalities, set(required_modalities))
     logger = get_logger()
     for name in regular:

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/common/normalize.py

@@ -98,7 +98,9 @@ def normalize_sky_mask(sky_mask) -> np.ndarray:
     """Normalise a sky mask to a 2-D boolean numpy array ``(H, W)``.
 
     Handles torch tensors (GPU loaders) and the ``(1, H, W)`` channel-first
-    layout produced by euler-loading's GPU loaders.
+    layout produced by euler-loading's GPU loaders. A non-boolean input may
+    only hold 0/1: casting class IDs to bool would mark every non-zero class
+    as sky.
     """
     mask = _to_numpy(sky_mask)
     # CHW → take first channel: (C, H, W) → (H, W)
@@ -109,4 +111,15 @@ def normalize_sky_mask(sky_mask) -> np.ndarray:
         mask = mask[0]
     if mask.ndim == 3 and mask.shape[-1] == 1:
         mask = mask[..., 0]
+    if mask.dtype != np.bool_ and np.any((mask != 0) & (mask != 1)):
+        values = np.unique(mask)
+        shown = ", ".join(str(value) for value in values[:8])
+        if values.size > 8:
+            shown += ", ..."
+        raise ValueError(
+            "semantic_segmentation must be a boolean sky mask, got a "
+            f"{mask.dtype} array with values [{shown}]. Load it with the "
+            "dataset's euler-loading sky_mask loader (the CLI selects it "
+            "automatically) or pass a boolean mask."
+        )
     return mask.astype(bool)

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/common/transform.py

@@ -20,6 +20,10 @@ class Transform(ABC):
         store per sample or once per hierarchy level; their role is inferred
         from the ds-crawler indexes (see
         :func:`~euler_preprocess.common.dataset.build_dataset`).
+        :attr:`MODALITY_FUNCTIONS` maps a modality to the euler-loading loader
+        function the transform needs, taken from the loader module its
+        dataset head names (``{"semantic_segmentation": "sky_mask"}`` reads a
+        class-segmentation index as sky masks).
 
     Output slots:
         Most transforms produce a single output (the *primary* slot, declared
@@ -32,6 +36,7 @@ class Transform(ABC):
 
     REQUIRED_MODALITIES: ClassVar[set[str]] = set()
     REQUIRED_HIERARCHICAL_MODALITIES: ClassVar[set[str]] = set()
+    MODALITY_FUNCTIONS: ClassVar[dict[str, str]] = {}
     SOURCE_MODALITY: ClassVar[str | None] = None
     OUTPUT_SLOT: ClassVar[str | None] = None
     OUTPUT_SLOTS: ClassVar[tuple[str, ...]] = ()

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/fog/atmospheric_light.py

@@ -124,7 +124,7 @@ class AtmosphericLightResolver:
         airlight_is_estimated = uses_estimated_airlight(al_spec)
         if airlight_is_estimated:
             raw_ls_base = normalize_atmospheric_light_torch(
-                estimated_airlight_t
+                estimated_airlight_t, allow_8bit=False
             ).squeeze(0)
         else:
             sampled_al = sample_value(al_spec, rng)
@@ -156,7 +156,7 @@ class AtmosphericLightResolver:
         airlight_is_estimated = uses_estimated_airlight(al_spec)
         if airlight_is_estimated:
             ls_base = self._estimate_batch_torch(rgb_batch, items, device, method)
-            ls_base = normalize_atmospheric_light_torch(ls_base)
+            ls_base = normalize_atmospheric_light_torch(ls_base, allow_8bit=False)
         else:
             ls_values = []
             for item in items:

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/fog/inference.py

@@ -58,7 +58,7 @@ def render_fog_image(
     mode: FogInferenceMode = "cpu",
     sample_id: str | None = None,
     sample_index: int = 0,
-    sky_class: Sequence[int | float] | None = (29, 0, 0),
+    sky_class: Sequence[int | float] | None = None,
 ) -> np.ndarray:
     """Render and return only the RGB image for one in-memory sample."""
 
@@ -89,14 +89,15 @@ def render_fog_sample(
     mode: FogInferenceMode = "cpu",
     sample_id: str | None = None,
     sample_index: int = 0,
-    sky_class: Sequence[int | float] | None = (29, 0, 0),
+    sky_class: Sequence[int | float] | None = None,
 ) -> FogInferenceResult:
     """Render fog/camera artifacts for a single in-memory sample.
 
     The call accepts the four modalities used by :class:`FogTransform` without
     requiring a dataset reader or output backend. ``semantic_segmentation`` may
     be either a boolean sky mask or an RGB semantic label map; RGB maps are
-    converted using ``sky_class`` which defaults to ``[29, 0, 0]``.
+    converted using ``sky_class``, the map's sky colour as its dataset head
+    declares it in ``skyclass`` (e.g. ``[29, 0, 0]`` for Real Drive Sim).
     """
 
     if mode not in ("cpu", "gpu"):

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/fog/logging.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import Sequence
+from typing import Any
 
 
 def log_config(
@@ -18,7 +20,14 @@ def log_config(
     resize_depth: bool,
     seed,
     contrast_threshold_default: float,
+    scenarios: Sequence[tuple[str, Any, Any]] = (),
+    scenario_scope: str = "sample",
 ) -> None:
+    """Log the resolved run settings.
+
+    *scenarios* holds ``(name, weight, model)`` per scenario profile, with
+    ``model`` ``None`` when the profile defers to ``selection``.
+    """
     logger.info("Loaded config from %s -> output %s", config_path, out_path)
     logger.info(
         "Device: requested=%s active=%s use_gpu=%s torch=%s",
@@ -36,6 +45,22 @@ def log_config(
         seed,
         contrast_threshold_default,
     )
+    if scenarios:
+        logger.info(
+            "Model selection: %d scenario_profiles (scope=%s) choose the model; "
+            "'selection' only applies to profiles without one",
+            len(scenarios),
+            scenario_scope,
+        )
+        for name, weight, model in scenarios:
+            logger.info(
+                "Scenario %s: weight=%s model=%s",
+                name,
+                weight,
+                "<selection>" if model is None else model,
+            )
+        if all(model is not None for _name, _weight, model in scenarios):
+            return
     selection = config.get("selection")
     if selection:
         mode = selection.get("mode", "fixed")

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/fog/models.py

@@ -160,18 +160,34 @@ def resolve_scattering_coefficient(
     )
 
 
-def normalize_atmospheric_light(value: np.ndarray) -> np.ndarray:
+def normalize_atmospheric_light(
+    value: np.ndarray, *, allow_8bit: bool = True
+) -> np.ndarray:
+    """Return ``value`` as an RGB triple clipped to ``[0, 1]``.
+
+    With *allow_8bit*, values above 1 are read as an 8-bit colour and divided
+    by 255. That only suits configured colours: estimated airlight comes from
+    ``[0, 1]`` RGB and may overshoot 1 slightly after the estimators' luminance
+    rescaling, so estimate callers pass ``allow_8bit=False`` to clip instead.
+    """
     value = np.asarray(value, dtype=np.float32)
     if value.ndim == 0:
         value = np.array([value, value, value], dtype=np.float32)
     if value.ndim != 1 or value.shape[0] != 3:
         raise ValueError("atmospheric_light must be scalar or length-3 list")
-    if value.max() > 1.0:
+    if allow_8bit and value.max() > 1.0:
         value = value / 255.0
     return np.clip(value, 0.0, 1.0)
 
 
-def normalize_atmospheric_light_torch(value: "torch.Tensor") -> "torch.Tensor":
+def normalize_atmospheric_light_torch(
+    value: "torch.Tensor", *, allow_8bit: bool = True
+) -> "torch.Tensor":
+    """Torch counterpart of :func:`normalize_atmospheric_light`.
+
+    The 8-bit check uses the maximum over every row, so stacked estimates must
+    pass ``allow_8bit=False`` or one overshooting sample rescales the batch.
+    """
     value_t = value.to(torch.float32)
     if value_t.ndim == 0:
         value_t = value_t.repeat(3)
@@ -180,7 +196,7 @@ def normalize_atmospheric_light_torch(value: "torch.Tensor") -> "torch.Tensor":
     if value_t.ndim != 2 or value_t.shape[-1] != 3:
         raise ValueError("atmospheric_light must be scalar or length-3 list")
     max_val = float(value_t.max().item()) if value_t.numel() else 0.0
-    if max_val > 1.0:
+    if allow_8bit and max_val > 1.0:
         value_t = value_t / 255.0
     return torch.clamp(value_t, 0.0, 1.0)
 
@@ -1653,7 +1669,9 @@ def apply_model(
     al_spec = model_cfg.get("atmospheric_light", "from_sky")
     airlight_is_estimated = uses_estimated_airlight(al_spec)
     if airlight_is_estimated:
-        raw_ls_base = normalize_atmospheric_light(estimated_airlight)
+        raw_ls_base = normalize_atmospheric_light(
+            estimated_airlight, allow_8bit=False
+        )
     else:
         sampled_al = sample_value(al_spec, rng)
         raw_ls_base = normalize_atmospheric_light(np.asarray(sampled_al))

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/fog/transform.py

@@ -301,6 +301,7 @@ class FogTransform(Transform):
 
     REQUIRED_MODALITIES: ClassVar[set[str]] = {"rgb", "depth", "semantic_segmentation"}
     REQUIRED_HIERARCHICAL_MODALITIES: ClassVar[set[str]] = set()
+    MODALITY_FUNCTIONS: ClassVar[dict[str, str]] = {"semantic_segmentation": "sky_mask"}
     SOURCE_MODALITY: ClassVar[str] = "rgb"
     OUTPUT_SLOT: ClassVar[str] = "rgb"
     OUTPUT_SLOTS: ClassVar[tuple[str, ...]] = (
@@ -399,6 +400,8 @@ class FogTransform(Transform):
             self.resize_depth_flag,
             self.seed,
             self.contrast_threshold_default,
+            scenarios=self._scenario_log_entries(),
+            scenario_scope=self._scenario_scope(),
         )
 
         airlight_method = self.config.get("airlight")
@@ -664,6 +667,42 @@ class FogTransform(Transform):
             name = config.get("name", config.get("id"))
         return None if name is None else str(name)
 
+    def _scenario_log_entries(self) -> list[tuple[str, Any, Any]]:
+        """Return ``(name, weight, model)`` per scenario profile for the config log."""
+        entries: list[tuple[str, Any, Any]] = []
+        for index, profile in enumerate(self.scenario_profiles):
+            payload = self._scenario_payload(profile)
+            model = next(
+                (
+                    payload[key]
+                    for key in ("model", "model_name", "fog_model")
+                    if key in payload
+                ),
+                None,
+            )
+            weight = profile.get(
+                "weight",
+                profile.get("profile_weight", profile.get("probability", 1.0)),
+            )
+            name = self._scenario_name(profile) or f"scenario_{index:03d}"
+            entries.append((name, weight, model))
+        return entries
+
+    def _scenario_scope(self) -> str:
+        """Describe how often a scenario profile is drawn, for the config log."""
+        if self.mode == "progressive":
+            return "progressive"
+        if (
+            self.use_gpu
+            and not self.augmentation_specs
+            and self.gpu_scenario_scope == "batch"
+        ):
+            return (
+                "batch, condition parameters per "
+                f"{self.gpu_condition_parameter_scope}"
+            )
+        return "sample"
+
     def _scenario_config_override(self, payload: dict[str, Any]) -> dict[str, Any]:
         return {
             key: value

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess/sky_depth/transform.py

@@ -31,6 +31,7 @@ class SkyDepthTransform(Transform):
     """
 
     REQUIRED_MODALITIES: ClassVar[set[str]] = {"depth", "semantic_segmentation"}
+    MODALITY_FUNCTIONS: ClassVar[dict[str, str]] = {"semantic_segmentation": "sky_mask"}
     SOURCE_MODALITY: ClassVar[str] = "depth"
     OUTPUT_SLOT: ClassVar[str] = "depth"
 
@@ -257,8 +258,7 @@ class SkyDepthTransform(Transform):
             log.warning(
                 "  SANITY CHECK: %.2f%% of pixels are marked as sky in sample "
                 "'%s'. That is suspiciously high — check that the mask is not "
-                "inverted or that `astype(bool)` is not accidentally treating "
-                "every non-zero class id as sky.",
+                "inverted and that the dataset head's `skyclass` names the sky.",
                 pct,
                 sample_id,
             )

euler_preprocess-3.16.0/README.md → euler_preprocess-3.17.0/euler_preprocess.egg-info/PKG-INFO

@@ -1,3 +1,23 @@
+Metadata-Version: 2.4
+Name: euler-preprocess
+Version: 3.17.0
+Summary: Physics-based preprocessing (fog, etc.) for RGB+depth datasets
+Author-email: Daniel Rothenpieler <rothenpielerdaniel@gmail.com>
+License-Expression: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: Pillow
+Requires-Dist: euler-loading>=2.29.0
+Requires-Dist: ds-crawler>=2.11.0
+Requires-Dist: euler-dataset-contract>=0.5.0
+Provides-Extra: gpu
+Requires-Dist: torch; extra == "gpu"
+Provides-Extra: progress
+Requires-Dist: tqdm; extra == "progress"
+Dynamic: license-file
+
 <!-- euler header — shared across the euler packages.
      Per package, change only: the <h1>, the tagline, and the badge URLs. -->
 <p align="center">
@@ -99,6 +119,15 @@ loader.
 | `sky-depth` | `depth`, `semantic_segmentation` | — |
 | `radial` | `depth`, `intrinsics` | — |
 
+`fog` and `sky-depth` need `semantic_segmentation` as a boolean sky mask. They load
+it with the `sky_mask` function of the euler-loading module that the index's
+dataset head names, so a `class_segmentation` index works as is: the sky is the
+head's `skyclass` (Synscapes `[23, 23, 23]`, Real Drive Sim `[29, 0, 0]`, VKITTI2
+`[90, 200, 255]`), and the log names the loader and the sky definition in use.
+Set `"function"` in the modality entry to use another loader function, or
+`"function": null` for the one the head declares. A non-binary mask raises an
+error rather than marking every non-zero class as sky.
+
 **Per-sample vs. hierarchical modalities.** You don't declare a modality's role:
 it is inferred from the ds-crawler indexes. A modality whose qualified IDs
 (hierarchy path plus file ID) match the transform's per-sample inputs is joined per
@@ -647,7 +676,8 @@ where sky depth is encoded as zero or infinity and needs a large finite value.
 }
 ```
 
-`sky_depth_value` defaults to `1000.0`.
+`sky_depth_value` defaults to `1000.0`. The sky mask comes from the dataset head as
+described under [Dataset Config](#dataset-config).
 
 CLI runs write a source-backed depth dataset mirroring the input depth modality's paths,
 filenames, extensions, and metadata. Standalone `SkyDepthTransform(...)` usage keeps the

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess.egg-info/SOURCES.txt

@@ -51,6 +51,7 @@ tests/test_dense_gloomy_daylight_config.py
 tests/test_fog_aux_outputs.py
 tests/test_fog_aware_auto_exposure.py
 tests/test_fog_cpu_gpu_parity.py
+tests/test_fog_input_normalization.py
 tests/test_fog_integration.py
 tests/test_modality_roles.py
 tests/test_radial.py

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/euler_preprocess.egg-info/requires.txt

@@ -1,6 +1,6 @@
 numpy
 Pillow
-euler-loading>=2.24.0
+euler-loading>=2.29.0
 ds-crawler>=2.11.0
 euler-dataset-contract>=0.5.0
 

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "euler-preprocess"
-version = "3.16.0"
+version = "3.17.0"
 description = "Physics-based preprocessing (fog, etc.) for RGB+depth datasets"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -16,7 +16,7 @@ authors = [
 dependencies = [
   "numpy",
   "Pillow",
-  "euler-loading>=2.24.0",
+  "euler-loading>=2.29.0",
   "ds-crawler>=2.11.0",
   "euler-dataset-contract>=0.5.0",
 ]

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/tests/test_fog_aux_outputs.py

@@ -1753,6 +1753,7 @@ def test_render_fog_sample_accepts_modalities_and_named_scenario() -> None:
         config=config,
         scenario_profile_name="dark_reference",
         mode="cpu",
+        sky_class=(29, 0, 0),
     )
     image = render_fog_image(
         rgb=rgb,
@@ -1767,6 +1768,7 @@ def test_render_fog_sample_accepts_modalities_and_named_scenario() -> None:
         config=config,
         scenario_profile_name="dark_reference",
         mode="cpu",
+        sky_class=(29, 0, 0),
     )
 
     assert result.scenario_name == "dark_reference"

euler_preprocess-3.17.0/tests/test_fog_input_normalization.py

@@ -0,0 +1,259 @@
+"""Regression tests for sky-mask and estimated-airlight normalisation."""
+
+import json
+
+import numpy as np
+import pytest
+from ds_crawler import DatasetWriter
+from euler_loading.loaders.cpu import real_drive_sim as rds
+from euler_loading.loaders.cpu.generic_dense_depth import write_depth
+
+from euler_preprocess.common.dataset import build_dataset
+from euler_preprocess.common.logging import get_logger
+from euler_preprocess.common.normalize import normalize_sky_mask
+from euler_preprocess.fog.atmospheric_light import AtmosphericLightResolver
+from euler_preprocess.fog.models import (
+    apply_model,
+    normalize_atmospheric_light,
+    resolve_model_config,
+)
+from euler_preprocess.fog.transform import FogTransform
+from euler_preprocess.sky_depth.transform import SkyDepthTransform
+
+try:
+    import torch
+except ImportError:
+    torch = None
+
+SKY = 23
+# Cityscapes-style label map: every pixel has a non-zero class ID.
+LABELS = np.array(
+    [
+        [SKY, SKY, SKY, SKY],
+        [SKY, 7, 7, 11],
+        [7, 7, 11, 11],
+        [7, 7, 11, 11],
+    ],
+    dtype=np.int64,
+)
+UNIFORM_CFG = {
+    "visibility_m": 100.0,
+    "atmospheric_light": "from_sky",
+    "airlight_dampening": False,
+}
+
+
+class TestNormalizeSkyMask:
+    def test_class_map_raises(self):
+        with pytest.raises(ValueError, match="sky_mask loader") as excinfo:
+            normalize_sky_mask(LABELS[None])
+        assert "7, 11, 23" in str(excinfo.value)
+
+    def test_rgb_class_map_raises(self):
+        # VKITTI2 GPU layout (3, H, W); vegetation (90, 240, 0) shares sky's red.
+        colours = np.zeros((3, 6, 6), dtype=np.int64)
+        colours[:, :2] = np.array([90, 200, 255])[:, None, None]
+        colours[:, 2:] = np.array([90, 240, 0])[:, None, None]
+        with pytest.raises(ValueError, match="boolean sky mask"):
+            normalize_sky_mask(colours)
+
+    def test_boolean_and_binary_masks_pass(self):
+        mask = LABELS == SKY
+        np.testing.assert_array_equal(normalize_sky_mask(mask), mask)
+        binary = mask.astype(np.uint8)[..., None]
+        np.testing.assert_array_equal(normalize_sky_mask(binary), mask)
+
+    @pytest.mark.skipif(torch is None, reason="torch not installed")
+    def test_gpu_loader_mask_passes(self):
+        mask = torch.from_numpy(LABELS == SKY)[None]
+        np.testing.assert_array_equal(normalize_sky_mask(mask), LABELS == SKY)
+
+
+def test_fog_transform_rejects_class_maps(tmp_path):
+    cfg = {
+        "airlight": "from_sky",
+        "device": "cpu",
+        "models": {"uniform": UNIFORM_CFG},
+        "selection": {"mode": "fixed", "model": "uniform"},
+    }
+    path = tmp_path / "fog.json"
+    path.write_text(json.dumps(cfg))
+    transform = FogTransform(config_path=str(path), out_path=str(tmp_path / "out"))
+    sample = {
+        "rgb": np.full(LABELS.shape + (3,), 0.5, dtype=np.float32),
+        "depth": np.full(LABELS.shape, 20.0, dtype=np.float32),
+        "semantic_segmentation": LABELS,
+        "id": "frame",
+    }
+    with pytest.raises(ValueError, match="sky_mask loader"):
+        transform.run([sample])
+
+
+# Real Drive Sim class IDs: 29 is sky, 31 the class a relabelled head declares.
+RDS_CLASSES = np.array([[29, 29, 7], [31, 7, 7]], dtype=np.uint8)
+FRAMES = ("Scene01/Camera_0/0000", "Scene01/Camera_0/0001")
+
+
+def _rds_dataset_config(tmp_path, skyclass, segmentation_entry=None) -> dict:
+    """Depth plus an RDS ``class_segmentation`` index whose head declares *skyclass*."""
+    depth = DatasetWriter(
+        tmp_path / "depth",
+        name="depth",
+        type="depth",
+        separator=None,
+        meta={"radial_depth": False, "scale_to_meters": 1.0, "range": [0, 1000]},
+        euler_loading={"loader": "generic_dense_depth", "function": "depth"},
+    )
+    segmentation = DatasetWriter(
+        tmp_path / "segmentation",
+        name="segmentation",
+        type="semantic_segmentation",
+        separator=None,
+        meta={"skyclass": skyclass},
+        euler_loading={"loader": "real_drive_sim", "function": "class_segmentation"},
+    )
+    for full_id in FRAMES:
+        name = full_id.rsplit("/", 1)[-1]
+        write_depth(str(depth.get_path(full_id, f"{name}.npy")), np.full((2, 3), 5.0, dtype=np.float32))
+        rds.write_class_segmentation(str(segmentation.get_path(full_id, f"{name}.png")), RDS_CLASSES)
+    depth.save_index()
+    segmentation.save_index()
+    entry = str(tmp_path / "segmentation")
+    if segmentation_entry is not None:
+        entry = {"path": entry, **segmentation_entry}
+    return {
+        "modalities": {"depth": str(tmp_path / "depth"), "semantic_segmentation": entry}
+    }
+
+
+def _run_sky_depth(tmp_path, config) -> np.ndarray:
+    dataset = build_dataset(
+        config,
+        SkyDepthTransform.REQUIRED_MODALITIES,
+        None,
+        SkyDepthTransform.MODALITY_FUNCTIONS,
+    )
+    cfg_path = tmp_path / "sky_depth.json"
+    cfg_path.write_text(json.dumps({"sky_depth_value": 900.0, "sanity_check_samples": 0}))
+    transform = SkyDepthTransform(str(cfg_path), str(tmp_path / "out"))
+    return np.load(transform.run(dataset)[0])
+
+
+class TestSkyMaskFromDatasetHead:
+    """The transforms read class-segmentation indexes through euler-loading's ``sky_mask``."""
+
+    def test_sky_comes_from_the_head_skyclass(self, tmp_path):
+        config = _rds_dataset_config(tmp_path, [29, 0, 0])
+        np.testing.assert_array_equal(
+            _run_sky_depth(tmp_path, config), np.where(RDS_CLASSES == 29, 900.0, 5.0)
+        )
+
+    def test_relabelled_head_moves_the_sky(self, tmp_path):
+        config = _rds_dataset_config(tmp_path, [31, 0, 0])
+        np.testing.assert_array_equal(
+            _run_sky_depth(tmp_path, config), np.where(RDS_CLASSES == 31, 900.0, 5.0)
+        )
+
+    def test_function_null_keeps_the_class_map_and_fails_loudly(self, tmp_path):
+        config = _rds_dataset_config(tmp_path, [29, 0, 0], {"function": None})
+        with pytest.raises(ValueError, match="boolean sky mask"):
+            _run_sky_depth(tmp_path, config)
+
+
+class TestEstimatedAirlightRange:
+    # A DCP-heuristic estimate whose luminance rescale overshoots one channel.
+    ESTIMATE = [0.9, 1.03, 0.98]
+    CLIPPED = [0.9, 1.0, 0.98]
+
+    def test_estimate_overshoot_is_clipped_not_rescaled(self):
+        result = normalize_atmospheric_light(np.array(self.ESTIMATE), allow_8bit=False)
+        np.testing.assert_allclose(result, self.CLIPPED, rtol=1e-6)
+
+    def test_configured_8bit_colour_is_still_rescaled(self):
+        result = normalize_atmospheric_light(np.array([200, 210, 220]))
+        np.testing.assert_allclose(result, np.array([200, 210, 220]) / 255.0, rtol=1e-6)
+
+    def test_cpu_model_keeps_overshooting_estimate(self):
+        _foggy, _beta, ls_base, _k_map, _ls_map = apply_model(
+            np.full((4, 4, 3), 0.5, dtype=np.float32),
+            np.full((4, 4), 20.0, dtype=np.float32),
+            "uniform",
+            resolve_model_config("uniform", {"uniform": UNIFORM_CFG}),
+            np.random.default_rng(0),
+            0.05,
+            np.array(self.ESTIMATE, dtype=np.float32),
+        )
+        np.testing.assert_allclose(ls_base, self.CLIPPED, rtol=1e-6)
+
+    @pytest.mark.skipif(torch is None, reason="torch not installed")
+    def test_torch_sample_keeps_overshooting_estimate(self):
+        _k_mean, ls_base = AtmosphericLightResolver("from_sky").resolve_model_torch(
+            model_cfg=resolve_model_config("uniform", {"uniform": UNIFORM_CFG}),
+            rng=np.random.default_rng(0),
+            contrast_threshold_default=0.05,
+            estimated_airlight_t=torch.tensor(self.ESTIMATE),
+            device=torch.device("cpu"),
+        )
+        torch.testing.assert_close(ls_base, torch.tensor(self.CLIPPED))
+
+    @pytest.mark.skipif(torch is None, reason="torch not installed")
+    def test_torch_batch_overshoot_leaves_other_samples_unchanged(self, monkeypatch):
+        resolver = AtmosphericLightResolver("from_sky")
+        estimates = torch.tensor([[0.5, 0.6, 0.7], self.ESTIMATE])
+        monkeypatch.setattr(resolver, "_estimate_batch_torch", lambda *a, **k: estimates)
+        model_cfg = resolve_model_config("uniform", {"uniform": UNIFORM_CFG})
+        items = [
+            {"model_cfg": model_cfg, "rng": np.random.default_rng(index)}
+            for index in range(2)
+        ]
+        _k_means, ls_base = resolver.resolve_uniform_batch_torch(
+            rgb_batch=torch.zeros(2, 4, 4, 3),
+            items=items,
+            device=torch.device("cpu"),
+            contrast_threshold_default=0.05,
+        )
+        torch.testing.assert_close(
+            ls_base, torch.tensor([[0.5, 0.6, 0.7], self.CLIPPED])
+        )
+
+
+class TestScenarioSelectionLog:
+    def _log_messages(self, tmp_path, caplog, scenario_profiles):
+        cfg = {
+            "airlight": "from_sky",
+            "device": "cpu",
+            "models": {"uniform": UNIFORM_CFG},
+            "selection": {"mode": "fixed", "model": "heterogeneous_k_ls"},
+            "scenario_profiles": scenario_profiles,
+        }
+        path = tmp_path / "scenarios.json"
+        path.write_text(json.dumps(cfg))
+        logger = get_logger()
+        logger.addHandler(caplog.handler)
+        try:
+            FogTransform(config_path=str(path), out_path=str(tmp_path / "out"))
+        finally:
+            logger.removeHandler(caplog.handler)
+        return caplog.messages
+
+    def test_profiles_with_models_do_not_log_fixed_model(self, tmp_path, caplog):
+        messages = self._log_messages(
+            tmp_path,
+            caplog,
+            [
+                {"name": "haze", "weight": 0.7, "model": "uniform"},
+                {"name": "clear", "weight": 0.3, "model": "uniform", "clear_weather": True},
+            ],
+        )
+        assert any("2 scenario_profiles (scope=sample)" in m for m in messages)
+        assert "Scenario haze: weight=0.7 model=uniform" in messages
+        assert not any("Model fixed" in m for m in messages)
+
+    def test_profile_without_model_defers_to_selection(self, tmp_path, caplog):
+        messages = self._log_messages(
+            tmp_path,
+            caplog,
+            [{"name": "default_model", "weight": 1.0}],
+        )
+        assert "Scenario default_model: weight=1.0 model=<selection>" in messages
+        assert "Model fixed: heterogeneous_k_ls" in messages

{euler_preprocess-3.16.0 → euler_preprocess-3.17.0}/tests/test_sky_depth.py

@@ -181,3 +181,14 @@ class TestSkyDepthTransform:
         transform = SkyDepthTransform(str(cfg_path), str(out_path), strict=False)
         paths = transform.run(samples)
         assert len(paths) == 2
+
+    def test_class_map_raises(self, tmp_path, sky_depth_config):
+        """A class-ID map must not be cast to bool (every non-zero class -> sky)."""
+        depth = np.ones((4, 4), dtype=np.float32)
+        labels = np.full((4, 4), 7, dtype=np.int64)
+        labels[0] = 23
+        samples = [{"id": "labels", "depth": depth, "semantic_segmentation": labels}]
+
+        transform = SkyDepthTransform(str(sky_depth_config), str(tmp_path / "output"))
+        with pytest.raises(ValueError, match="sky_mask loader"):
+            transform.run(samples)