evaluators 2.1.4__tar.gz → 3.0.0__tar.gz

{evaluators-2.1.4 → evaluators-3.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: evaluators
-Version: 2.1.4
+Version: 3.0.0
 Summary: Various scene understanding and perception evaluation metrics.
 Author: Kurt Stolle
 Author-email: Kurt Stolle <kurt@computer.org>
@@ -27,6 +27,7 @@ Requires-Dist: pydantic
 Requires-Dist: opencv-python>=4.13.0.92
 Requires-Dist: omegaconf>=2.3.0
 Requires-Dist: hydra-zen>=0.16.0
+Requires-Dist: scipy>=1.14.0
 Requires-Dist: blosc2>=4.1.2 ; extra == 'blosc2'
 Requires-Dist: lightning>=2.6.1 ; extra == 'lightning'
 Requires-Dist: lmdb>=2.2.0 ; extra == 'lmdb'
@@ -103,7 +104,7 @@ from evaluators.metrics.domain.segmentation import SemanticMetric
 dataset = CityscapesDataset(...)
 metric = SemanticMetric(
     num_classes=19,
-    target_source=dataset, 
+    target_source=dataset,
 )
 
 ### Define the explicit memory schema
@@ -144,12 +145,12 @@ for batch in dataloader: # Model forward pass
             "frame_index": batch["frame_index"]
         }
     )
-    
+
 # Finalize
 writer.close()
 ```
 
-    > Note: `sequence_id` and `frame_index` must be `torch.int64`. 
+    > Note: `sequence_id` and `frame_index` must be `torch.int64`.
 
 ##### 4. Finalize and compute
 

{evaluators-2.1.4 → evaluators-3.0.0}/README.md

@@ -62,7 +62,7 @@ from evaluators.metrics.domain.segmentation import SemanticMetric
 dataset = CityscapesDataset(...)
 metric = SemanticMetric(
     num_classes=19,
-    target_source=dataset, 
+    target_source=dataset,
 )
 
 ### Define the explicit memory schema
@@ -103,12 +103,12 @@ for batch in dataloader: # Model forward pass
             "frame_index": batch["frame_index"]
         }
     )
-    
+
 # Finalize
 writer.close()
 ```
 
-    > Note: `sequence_id` and `frame_index` must be `torch.int64`. 
+    > Note: `sequence_id` and `frame_index` must be `torch.int64`.
 
 ##### 4. Finalize and compute
 

{evaluators-2.1.4 → evaluators-3.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "evaluators"
-version = "2.1.4"
+version = "3.0.0"
 description = "Various scene understanding and perception evaluation metrics."
 readme = "README.md"
 requires-python = ">=3.13, <3.14"
@@ -35,6 +35,7 @@ dependencies = [
   "opencv-python>=4.13.0.92",
   "omegaconf>=2.3.0",
   "hydra-zen>=0.16.0",
+  "scipy>=1.14.0",
 ]
 
 [project.optional-dependencies]
@@ -117,6 +118,10 @@ ignore = [
   "SIM118",  # usage of 'x in dict.keys()' -- conflicts with `tensordict`.
   "PLR2004", # magic numbers
   "PLW2901", # for loop variable overwritten
+  # Lazy/deferred imports are a deliberate, pervasive pattern (optional deps:
+  # matplotlib, codecs, heavy adapters) used to keep import and CLI startup
+  # cheap; PLC0415 (import-should-be-at-top) fights that design codebase-wide.
+  "PLC0415",
   "D212",
   "D203",
   "D107",    # docstring on __init__
@@ -209,6 +214,10 @@ include = ["sources", "tests", "notebooks", "scripts"]
 
 [tool.pytest.ini_options]
 timeout = 10
+# Vendored reference implementations under third_party/ ship their own
+# environment-dependent test suites (e.g. TrackEval); never collect them as
+# part of this project's tests. Mirrors the ruff `exclude = ["third_party"]`.
+addopts = "--ignore=third_party"
 markers = [
   "integration: integration tests that exercise real pretrained model inference end-to-end (may require network for first-time weight download).",
   "slow: tests with high wall-clock cost.",

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/_engine.py

@@ -272,7 +272,7 @@ def _save_plots(
     force: bool,
 ) -> None:
     """Call ``plot()`` on each metric and save the figure to *out_path*."""
-    import matplotlib.pyplot as plt
+    import matplotlib.pyplot as plt  # lazy: matplotlib is optional
 
     for metric in metrics:
         name = metric.__class__.__name__

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/_streams.py

@@ -216,9 +216,7 @@ class MetricStream(nn.Module):
 
         # 2. Plumbing / Renaming
         if self.input_map:
-            # We invert the map to {model_key: metric_key} for the rename operation
-            # rename_map = {v: k for k, v in self.input_map.items()}
-            # filtered_batch = filtered_batch.rename_keys_(rename_map, strict=False)
+            # We map model keys to metric keys as defined in the input map.
             for metric_key, model_key in self.input_map.items():
                 if model_key in filtered_batch.keys():
                     filtered_batch.rename_key_(model_key, metric_key)

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/benchmarks/sys_info.py

@@ -139,7 +139,7 @@ def get_machine_info() -> MachineInfo:
             # CPU Info
             try:
                 cpu_info = subprocess.check_output(
-                    "lscpu", shell=True, stderr=subprocess.DEVNULL
+                    ["lscpu"], stderr=subprocess.DEVNULL
                 ).decode()
                 for line in cpu_info.split("\n"):
                     if "Model name:" in line:
@@ -166,8 +166,14 @@ def get_machine_info() -> MachineInfo:
             try:
                 mnt_out = (
                     subprocess.check_output(
-                        f"findmnt -n -o SOURCE,FSTYPE --target {current_dir}",
-                        shell=True,
+                        [
+                            "findmnt",
+                            "-n",
+                            "-o",
+                            "SOURCE,FSTYPE",
+                            "--target",
+                            current_dir,
+                        ],
                         stderr=subprocess.DEVNULL,
                     )
                     .decode()
@@ -184,8 +190,7 @@ def get_machine_info() -> MachineInfo:
                         try:
                             parent = (
                                 subprocess.check_output(
-                                    f"lsblk -no PKNAME {target_dev}",
-                                    shell=True,
+                                    ["lsblk", "-no", "PKNAME", target_dev],
                                     stderr=subprocess.DEVNULL,
                                 )
                                 .decode()
@@ -204,8 +209,7 @@ def get_machine_info() -> MachineInfo:
                     try:
                         model_lines = (
                             subprocess.check_output(
-                                f"lsblk -no MODEL {target_dev}",
-                                shell=True,
+                                ["lsblk", "-no", "MODEL", target_dev],
                                 stderr=subprocess.DEVNULL,
                             )
                             .decode()

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/cli/memory.py

@@ -289,9 +289,11 @@ def handle_validate(args):
             status.update(f"[bold green]Validating {vid_id}...")
             if args.quick:
                 # Just check if fragment files exist
-                for frag in entry["fragments"]:
-                    if not (path / frag["chunk_id"]).exists():
-                        errors.append((vid_id, f"Chunk {frag['chunk_id']} missing"))
+                errors.extend(
+                    (vid_id, f"Chunk {frag['chunk_id']} missing")
+                    for frag in entry["fragments"]
+                    if not (path / frag["chunk_id"]).exists()
+                )
                 continue
 
             try:

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/integrations/lightning.py

@@ -10,6 +10,7 @@ Exports
 - ``DistributedInferenceWriter``: For Predict mode (Save only).
 """
 
+import contextlib
 import gc
 import logging
 import shutil
@@ -24,6 +25,7 @@ from evaluators.memory import (
     TemporalWriterProtocol,
     save_evaluation_config,
 )
+from evaluators.types import MetricResult
 from evaluators.utils import comms
 from lightning import Callback, LightningModule, Trainer
 
@@ -55,15 +57,23 @@ def _default_evaluate_streams(
     pl_module: LightningModule | None = None,
 ) -> dict[str, dict[str, dict[str, float]]]:
     """Default post-predict evaluator using offline evaluation."""
-    del trainer, pl_module
+    del trainer
     artifact_root = Path(artifact_root)
     all_results: dict[str, dict[str, dict[str, float]]] = {}
 
+    # Pick the trainer's compute device for metric ``.update`` calls.
+    # ``run_offline_evaluation`` defaults to CPU if ``device`` is
+    # ``None``; the GPU dispatch shaves ~12 min off SemKITTI-DVPS
+    # finalize.  See the ``DistributedEvaluationCallback._finalize_epoch``
+    # callsite for the longer rationale.
+    device = pl_module.device if pl_module is not None else None
+
     for stream in streams:
         stream_results = run_offline_evaluation(
             metrics=stream.metrics,
             artifact_dir=artifact_root / stream.name,
             show_progress=False,
+            device=device,
         )
         all_results[stream.name] = stream_results
 
@@ -212,16 +222,18 @@ class DistributedEvaluationCallback(_DistributedWriterBase):
     def on_validation_epoch_end(
         self, trainer: Trainer, pl_module: LightningModule
     ) -> None:
-        self._finalize_epoch(pl_module, "val")
+        self._finalize_epoch(trainer, pl_module, "val")
 
     def on_test_epoch_start(self, trainer: Trainer, pl_module: LightningModule) -> None:
         self._setup_streams(pl_module)
 
     def on_test_epoch_end(self, trainer: Trainer, pl_module: LightningModule) -> None:
-        self._finalize_epoch(pl_module, "test")
+        self._finalize_epoch(trainer, pl_module, "test")
 
-    def _finalize_epoch(self, pl_module: LightningModule, split: str) -> None:
-        """Closes writers, runs metrics, and logs results to Lightning.
+    def _finalize_epoch(
+        self, trainer: Trainer, pl_module: LightningModule, split: str
+    ) -> None:
+        """Close writers, run metrics, log scalars and plots to Lightning.
 
         Writer-side memory maps are released before ``run_offline_evaluation``
         opens reader-side maps over the same files; otherwise the two sets
@@ -238,6 +250,17 @@ class DistributedEvaluationCallback(_DistributedWriterBase):
                     metrics=stream.metrics,
                     artifact_dir=stream_dir,
                     show_progress=False,
+                    # Route per-frame metric ``.update`` to the same
+                    # device the trainer is using.  Without this,
+                    # ``run_offline_evaluation`` defaults to ``None`` →
+                    # CPU, which made full-validation metric finalize
+                    # take ~12 min on the SemKITTI-DVPS val split
+                    # (255 frames, PQ + DVPQ {k=1,5,10,20} +
+                    # SemanticMetric + DepthMetric).  Panopticapi PQ
+                    # matching is still CPU-only (pure Python), but
+                    # the tensor reductions for the other three
+                    # metrics + the DVPQ window stacks move to GPU.
+                    device=pl_module.device,
                 )
 
                 if comms.get_rank() != 0:
@@ -249,7 +272,13 @@ class DistributedEvaluationCallback(_DistributedWriterBase):
                         log_key = f"{split}/{stream_name}/{metric_cls}/{key}"
                         pl_module.log(log_key, val, sync_dist=True)
 
-                # 2. Print Summary Tables (Interactive/Human-Friendly)
+                # 2. Per-metric plots (when the metric implements ``plot()``
+                #    and at least one Lightning logger supports image upload).
+                self._log_metric_plots(
+                    trainer, pl_module, split, stream_name, stream, stream_results
+                )
+
+                # 3. Print Summary Tables (Interactive/Human-Friendly)
                 if sys.stdout.isatty():
                     self._print_stream_summary(stream_name, stream)
 
@@ -263,6 +292,82 @@ class DistributedEvaluationCallback(_DistributedWriterBase):
             self._streams = {}
             gc.collect()
 
+    @staticmethod
+    def _log_metric_plots(
+        trainer: Trainer,
+        pl_module: LightningModule,
+        split: str,
+        stream_name: str,
+        stream: MetricStream,
+        stream_results: dict[str, dict[str, float]],
+    ) -> None:
+        """Render each metric's ``plot()`` and push the figure to every
+        Lightning logger that exposes ``log_image`` (e.g. ``WandbLogger``).
+
+        Iterates ``trainer.loggers`` so multi-logger setups (such as
+        ``[CSVLogger, WandbLogger]``) are handled correctly. Quietly skips
+        metrics that have no ``plot`` method, return None, or raise during
+        rendering; observability failures are not load-bearing for training
+        and must not crash the validation epoch.
+
+        Calls ``plot(val=...)`` with a precomputed :class:`MetricResult` so
+        the metric does not re-invoke ``compute()`` on state that
+        ``run_offline_evaluation`` has already consumed.
+        """
+        image_loggers = [
+            logger_obj
+            for logger_obj in getattr(trainer, "loggers", []) or []
+            if callable(getattr(logger_obj, "log_image", None))
+        ]
+        if not image_loggers:
+            return
+
+        for metric in stream.metrics:
+            plot_fn = getattr(metric, "plot", None)
+            if plot_fn is None or not callable(plot_fn):
+                continue
+
+            metric_name = metric.__class__.__name__
+            scores = stream_results.get(metric_name)
+            if scores is None:
+                continue
+
+            val = MetricResult.from_dict(scores)
+            try:
+                result = plot_fn(val=val)
+            except Exception:
+                logger.exception(
+                    "metric %s.plot() raised; skipping plot upload", metric_name
+                )
+                continue
+            if result is None:
+                continue
+
+            fig = result[0] if isinstance(result, tuple) else result
+            try:
+                for image_logger in image_loggers:
+                    try:
+                        image_logger.log_image(
+                            key=f"{split}/{stream_name}/{metric_name}/plot",
+                            images=[fig],
+                            step=getattr(pl_module, "global_step", None),
+                        )
+                    except Exception:
+                        logger.exception(
+                            "log_image failed for %s; skipping plot upload",
+                            metric_name,
+                        )
+            finally:
+                close_fn = getattr(fig, "close", None)
+                if close_fn is not None and callable(close_fn):
+                    with contextlib.suppress(Exception):
+                        close_fn()
+                else:
+                    with contextlib.suppress(Exception):
+                        import matplotlib.pyplot as plt
+
+                        plt.close(fig)
+
     @staticmethod
     def _print_stream_summary(stream_name: str, stream: MetricStream) -> None:
         from rich.console import Console

evaluators-3.0.0/sources/evaluators/io/__init__.py

@@ -0,0 +1,7 @@
+"""I/O readers for tracking-evaluation file formats (MOTChallenge, ...)."""
+
+from __future__ import annotations
+
+from .mot_challenge import read_mot_sequence
+
+__all__ = ["read_mot_sequence"]

evaluators-3.0.0/sources/evaluators/io/mot_challenge.py

@@ -0,0 +1,136 @@
+"""MOTChallenge ground-truth / tracker text-file reader.
+
+Parses MOTChallenge CSV rows
+``frame, id, bb_left, bb_top, bb_width, bb_height, conf, class, visibility``
+for both the gt file and a tracker results file, and returns a list of
+:class:`~evaluators.metrics.domain.tracking._preprocess.RawFrame` (one per
+timestep) ready for the MOTChallenge preprocessor.
+
+This module is I/O only; it does no scoring. In MOT gt files the 7th column
+(``conf``) is the 0/1 "consider" flag, exposed here as ``gt_zero_marked``; in
+tracker files the same column is the detection confidence.
+"""
+
+from __future__ import annotations
+
+import configparser
+import csv
+from pathlib import Path
+
+import numpy as np
+
+from evaluators.metrics.domain.tracking._preprocess import RawFrame
+
+__all__ = ["read_mot_sequence", "read_seq_length"]
+
+# Column layout (0-based) of a MOTChallenge CSV row.
+_COL_FRAME = 0
+_COL_ID = 1
+_COL_BOX = slice(2, 6)  # bb_left, bb_top, bb_width, bb_height (xywh)
+_COL_CONF = 6  # tracker confidence OR gt zero-marked flag
+_COL_CLASS = 7
+_MIN_COLS = 7  # frame..conf are always required; class defaults to 1 if absent.
+
+
+def _read_rows(path: Path) -> dict[int, list[list[float]]]:
+    """Parse a MOTChallenge file into ``{frame -> list of float rows}``."""
+    by_frame: dict[int, list[list[float]]] = {}
+    text = Path(path).read_text()
+    reader = csv.reader(text.splitlines())
+    for raw in reader:
+        row = [c for c in raw if c.strip() != ""]
+        if len(row) < _MIN_COLS:
+            continue
+        values = [float(c) for c in row]
+        frame = int(values[_COL_FRAME])
+        by_frame.setdefault(frame, []).append(values)
+    return by_frame
+
+
+def _frame_arrays(
+    rows: list[list[float]],
+) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """Convert raw float rows for one frame into ``(boxes, ids, classes, conf)``."""
+    if not rows:
+        return (
+            np.zeros((0, 4), dtype=np.float64),
+            np.zeros((0,), dtype=np.int64),
+            np.zeros((0,), dtype=np.int64),
+            np.zeros((0,), dtype=np.float64),
+        )
+    data = np.asarray(rows, dtype=np.float64)
+    boxes = data[:, _COL_BOX].astype(np.float64)
+    ids = data[:, _COL_ID].astype(np.int64)
+    conf = data[:, _COL_CONF].astype(np.float64)
+    if data.shape[1] > _COL_CLASS:
+        classes = data[:, _COL_CLASS].astype(np.int64)
+    else:
+        classes = np.ones_like(ids)
+    return boxes, ids, classes, conf
+
+
+def read_seq_length(seqinfo: str | Path) -> int:
+    """Read ``seqLength`` from a MOTChallenge ``seqinfo.ini``."""
+    parser = configparser.ConfigParser()
+    parser.read(seqinfo)
+    return int(parser["Sequence"]["seqLength"])
+
+
+def read_mot_sequence(
+    gt_path: str | Path,
+    ts_path: str | Path,
+    *,
+    seqinfo: str | Path | None = None,
+) -> list[RawFrame]:
+    """Read a MOTChallenge gt + tracker sequence into per-frame raw data.
+
+    Parameters
+    ----------
+    gt_path
+        Path to the ground-truth file (``gt.txt``).
+    ts_path
+        Path to the tracker results file (``det.txt`` / results).
+    seqinfo
+        Optional path to ``seqinfo.ini``. If given, frames are padded to
+        ``seqLength`` (trailing empty frames become empty arrays). If absent, the
+        number of frames is the maximum frame index present across either file.
+
+    Returns
+    -------
+    list of RawFrame
+        One :class:`RawFrame` per timestep, in frame order (index ``t`` is file
+        frame ``t + 1``). ``gt_zero_marked`` is taken from the gt ``conf`` column;
+        tracker ``conf`` becomes ``tracker_confidences``.
+    """
+    gt_by_frame = _read_rows(Path(gt_path))
+    ts_by_frame = _read_rows(Path(ts_path))
+
+    if seqinfo is not None:
+        num_frames = read_seq_length(seqinfo)
+    else:
+        max_gt = max(gt_by_frame, default=0)
+        max_ts = max(ts_by_frame, default=0)
+        num_frames = max(max_gt, max_ts)
+
+    frames: list[RawFrame] = []
+    for t in range(num_frames):
+        file_frame = t + 1
+        gt_boxes, gt_ids, gt_classes, gt_conf = _frame_arrays(
+            gt_by_frame.get(file_frame, [])
+        )
+        ts_boxes, ts_ids, ts_classes, ts_conf = _frame_arrays(
+            ts_by_frame.get(file_frame, [])
+        )
+        frames.append(
+            RawFrame(
+                gt_boxes=gt_boxes,
+                gt_ids=gt_ids,
+                gt_classes=gt_classes,
+                gt_zero_marked=gt_conf.astype(np.int64),
+                tracker_boxes=ts_boxes,
+                tracker_ids=ts_ids,
+                tracker_classes=ts_classes,
+                tracker_confidences=ts_conf,
+            )
+        )
+    return frames

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/memory/backends/_registry.py

@@ -113,6 +113,9 @@ def _validate_non_lmdb_chunks(artifact_dir: Path, chunk_ids: set[str]) -> None:
             raise FileNotFoundError(msg)
 
 
+# Backend reader imports are deferred to per-backend dispatchers so importing
+# the registry does not pull every backend's optional dependencies (lmdb,
+# torch.cuda.gds, etc.) at module load.
 def _open_lmdb_sequence(fragments: Sequence[interfaces.Fragment], artifact_dir: Path):
     from .lmdb._reader import LMDBVirtualSequence
 

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/memory/backends/gds/writer.py

@@ -40,7 +40,7 @@ class GDSTemporalWriter:
     ) -> None:
         # Check for torch.cuda.gds support
         try:
-            import torch.cuda.gds
+            import torch.cuda.gds  # optional CUDA submodule probe
 
             if not torch.cuda.gds.is_available():
                 msg = "NVIDIA GPUDirect Storage (GDS) is not available."

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/memory/backends/memmap/_buffers.py

@@ -72,8 +72,8 @@ class RaggedBuffer:
             filename=directory / f"{filename_prefix}_vals.memmap",
         )
 
-        # 2. Offsets Buffer (The index)
-        # Stores [start_0, start_1, ..., start_N, total_end]
+        # 2. Offsets Buffer (The index).
+        # Stores the start and end pointers for each row.
         # We need capacity_rows + 1 to store the final end pointer.
         self.offsets = MemoryMappedTensor.empty(
             (capacity_rows + 1,),

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/memory/backends/memmap/_writer.py

@@ -6,6 +6,7 @@ both fixed-size dense tensors and codec-encoded variable-length payloads under
 one implementation.
 """
 
+import contextlib
 import gc
 import json
 from pathlib import Path
@@ -335,10 +336,8 @@ class MemmapTemporalWriter:
 
     def __del__(self) -> None:
         # Best-effort cleanup if close() was skipped due to an exception path.
-        try:
+        with contextlib.suppress(Exception):
             self._release_buffers()
-        except Exception:
-            pass
 
     def _validate_special_key(self, key: str, tensor: object) -> None:
         if not torch.is_tensor(tensor):

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/memory/codecs/_blosc.py

@@ -99,7 +99,7 @@ class BloscCodec:
         )
         if isinstance(packed, int):
             msg = "Blosc returned file-size integer, expected in-memory bytes cframe"
-            raise ValueError(msg)
+            raise TypeError(msg)
         return packed
 
     @classmethod

{evaluators-2.1.4 → evaluators-3.0.0}/sources/evaluators/metrics/domain/depth.py

@@ -485,7 +485,7 @@ class DepthMetric(BaseMetric):
             The matplotlib figure and axes.
 
         """
-        import matplotlib.pyplot as plt
+        import matplotlib.pyplot as plt  # lazy: matplotlib optional
 
         result = val if val is not None else self.compute()
         d = result.to_dict() if hasattr(result, "to_dict") else dict(result)
@@ -579,7 +579,7 @@ class DepthMetric(BaseMetric):
             Depth metrics table.
 
         """
-        import pandas as pd
+        import pandas as pd  # lazy: pandas optional
 
         result = val if val is not None else self.compute()
         d = result.to_dict() if hasattr(result, "to_dict") else dict(result)