evaluators 2.1.1__tar.gz → 2.1.3__tar.gz

{evaluators-2.1.1 → evaluators-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: evaluators
-Version: 2.1.1
+Version: 2.1.3
 Summary: Various scene understanding and perception evaluation metrics.
 Author: Kurt Stolle
 Author-email: Kurt Stolle <kurt@computer.org>
@@ -23,14 +23,20 @@ Requires-Dist: tensordict>=0.11.0
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: matplotlib>=3.9.0
 Requires-Dist: rich>=13.9.4
-Requires-Dist: lmdb
-Requires-Dist: tifffile
 Requires-Dist: pydantic
 Requires-Dist: opencv-python>=4.13.0.92
-Requires-Dist: imagecodecs>=2026.3.6
+Requires-Dist: omegaconf>=2.3.0
+Requires-Dist: hydra-zen>=0.16.0
+Requires-Dist: blosc2>=4.1.2 ; extra == 'blosc2'
 Requires-Dist: lightning>=2.6.1 ; extra == 'lightning'
+Requires-Dist: lmdb>=2.2.0 ; extra == 'lmdb'
+Requires-Dist: imagecodecs>=2026.3.6 ; extra == 'tiff'
+Requires-Dist: tifffile>=2026.3.3 ; extra == 'tiff'
 Requires-Python: >=3.13, <3.14
+Provides-Extra: blosc2
 Provides-Extra: lightning
+Provides-Extra: lmdb
+Provides-Extra: tiff
 Description-Content-Type: text/markdown
 
 # Scalable Distributed Evaluation for Computer Vision
@@ -219,10 +225,10 @@ The evaluation pipeline consists of three stages.
    Each GPU writes predictions to locally sharded files (e.g. `.memmap` or `.lmdb`). No communication occurs.
 2. **Schedule.**  
    A synchronization barrier is reached. The main process aggregates metadata manifests from all shards to build a Global Catalog. It partitions the workload (videos) among workers using a greedy strategy to balance duration.
-3. **Compute.**  
+3. **Compute.**
    Workers iterate through their assigned Virtual Sequences. Data is streamed from disk, processed by torchmetrics, and reduced globally.
 
-See [ARCHITECTURE.md](ARCHITECTURE.md) for detailed diagrams and design principles.
+See [OFFLINE_EVALUATION.md](OFFLINE_EVALUATION.md) for detailed usage and design principles.
 
 ---
 

{evaluators-2.1.1 → evaluators-2.1.3}/README.md

@@ -184,10 +184,10 @@ The evaluation pipeline consists of three stages.
    Each GPU writes predictions to locally sharded files (e.g. `.memmap` or `.lmdb`). No communication occurs.
 2. **Schedule.**  
    A synchronization barrier is reached. The main process aggregates metadata manifests from all shards to build a Global Catalog. It partitions the workload (videos) among workers using a greedy strategy to balance duration.
-3. **Compute.**  
+3. **Compute.**
    Workers iterate through their assigned Virtual Sequences. Data is streamed from disk, processed by torchmetrics, and reduced globally.
 
-See [ARCHITECTURE.md](ARCHITECTURE.md) for detailed diagrams and design principles.
+See [OFFLINE_EVALUATION.md](OFFLINE_EVALUATION.md) for detailed usage and design principles.
 
 ---
 

{evaluators-2.1.1 → evaluators-2.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "evaluators"
-version = "2.1.1"
+version = "2.1.3"
 description = "Various scene understanding and perception evaluation metrics."
 readme = "README.md"
 requires-python = ">=3.13, <3.14"
@@ -31,15 +31,17 @@ dependencies = [
   "numpy>=2.0.0",
   "matplotlib>=3.9.0",
   "rich>=13.9.4",
-  "lmdb",
-  "tifffile",
   "pydantic",
   "opencv-python>=4.13.0.92",
-  "imagecodecs>=2026.3.6",
+  "omegaconf>=2.3.0",
+  "hydra-zen>=0.16.0",
 ]
 
 [project.optional-dependencies]
 lightning = ["lightning>=2.6.1"]
+lmdb = ["lmdb>=2.2.0"]
+blosc2 = ["blosc2>=4.1.2"]
+tiff = ["imagecodecs>=2026.3.6", "tifffile>=2026.3.3"]
 
 [dependency-groups]
 dev = [
@@ -98,15 +100,23 @@ select = [
   "UP",   # pyupgrade
 ]
 ignore = [
+  "SIM118",  # usage of 'x in dict.keys()' -- conflicts with `tensordict`.
   "PLR2004", # magic numbers
   "PLW2901", # for loop variable overwritten
   "D212",
   "D203",
   "D107",    # docstring on __init__
+  "PT011",   # pytest.raises too broad (restrictive)
+
+
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"tests/*" = ["D", "T"]
+"tests/*" = [
+  "D",
+  "T",
+  "E402", # Module level import not at top of file
+]
 
 [tool.ruff.format]
 docstring-code-format = true

{evaluators-2.1.1 → evaluators-2.1.3}/sources/evaluators/__init__.py

@@ -22,33 +22,39 @@ from typing import TYPE_CHECKING
 from ._engine import run_offline_evaluation
 from ._streams import BackendInterface, MetricStream, UnboundStreamWarning
 from .memory import (
-    EncodedMemmapTemporalWriter,
     LMDBTemporalWriter,
-    MemorySchema,
     MemmapTemporalWriter,
+    MemorySchema,
     TensorField,
+    deserialize_metrics,
+    load_evaluation_config,
+    save_evaluation_config,
+    serialize_metrics,
 )
 
 if TYPE_CHECKING:
     __version__: str
     __all__ = [
         "BackendInterface",
-        "EncodedMemmapTemporalWriter",
         "LMDBTemporalWriter",
-        "MemorySchema",
         "MemmapTemporalWriter",
+        "MemorySchema",
         "MetricStream",
         "TensorField",
         "UnboundStreamWarning",
         "__version__",
+        "deserialize_metrics",
+        "load_evaluation_config",
         "run_offline_evaluation",
+        "save_evaluation_config",
+        "serialize_metrics",
     ]
 
 
 # 2. Runtime Lazy Loader
 def __getattr__(name: str) -> str:
     if name == "__version__":
-        from importlib.metadata import PackageNotFoundError, version  # noqa: PLC0415
+        from importlib.metadata import PackageNotFoundError, version
 
         try:
             return version("evaluators")

{evaluators-2.1.1 → evaluators-2.1.3}/sources/evaluators/_engine.py

@@ -18,7 +18,7 @@ from pathlib import Path
 import rich.progress
 import torch
 
-from evaluators.memory import MemmapVirtualSequence
+from evaluators.memory.backends._registry import open_virtual_sequence
 from evaluators.scheduling import CatalogBuilder, WorkPartitioner
 from evaluators.types import BaseMetric, MetricResult
 from evaluators.utils import comms
@@ -192,7 +192,7 @@ def _evaluate_metrics(
 
     for i, item in enumerate(work_items):
         fragments = item.entry["fragments"]
-        sequence = MemmapVirtualSequence(fragments, artifact_path)
+        sequence = open_virtual_sequence(artifact_path, item.sequence_id, fragments)
 
         if show_progress:
             with rich.progress.Progress() as progress:
@@ -265,7 +265,7 @@ def _save_plots(
     force: bool,
 ) -> None:
     """Call ``plot()`` on each metric and save the figure to *out_path*."""
-    import matplotlib.pyplot as plt  # noqa: PLC0415
+    import matplotlib.pyplot as plt
 
     for metric in metrics:
         name = metric.__class__.__name__

{evaluators-2.1.1 → evaluators-2.1.3}/sources/evaluators/_streams.py

@@ -30,6 +30,7 @@ from tensordict import TensorDict
 from torch import nn
 
 from evaluators.types import BaseMetric
+
 from .memory.schema import MemorySchema
 
 __all__ = ["BackendInterface", "MetricStream", "UnboundStreamWarning"]

evaluators-2.1.3/sources/evaluators/cli/compute.py

@@ -0,0 +1,167 @@
+"""
+CLI Module: Evaluate.
+
+Runs offline evaluation on stored inference results using the config saved in
+the artifact directory.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import Any, cast
+
+import rich.console
+import rich.pretty
+import torch
+from evaluators import run_offline_evaluation
+from evaluators.memory import deserialize_metrics, load_evaluation_config
+from omegaconf import DictConfig, ListConfig
+
+logger = logging.getLogger(__name__)
+
+
+def setup_parser() -> argparse.ArgumentParser:
+    """Define arguments for the compute subcommand."""
+    parser = argparse.ArgumentParser(
+        prog="evaluators compute",
+        description="Run offline evaluation on stored inference results.",
+    )
+
+    parser.add_argument(
+        "artifact_dir",
+        type=str,
+        help="Path to directory containing stored inference results.",
+    )
+
+    parser.add_argument(
+        "--output-dir",
+        "-o",
+        type=str,
+        default=None,
+        help="Directory to save metric results and plots.",
+    )
+
+    parser.add_argument(
+        "--plot",
+        action="store_true",
+        help="Generate and save visualization plots.",
+    )
+
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Force recomputation of cached metrics.",
+    )
+
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="cuda" if torch.cuda.is_available() else "cpu",
+        help="Device for computation (default: cuda if available).",
+    )
+
+    parser.add_argument(
+        "--quiet",
+        "-q",
+        action="store_true",
+        help="Suppress progress bars.",
+    )
+
+    return parser
+
+
+def _load_from_config_file(config_path: str | Path) -> list[Any]:
+    """Load metrics from serialized YAML metric definitions in config file."""
+    config = load_evaluation_config(config_path)
+
+    raw_specs = config.get("metrics", [])
+    if not raw_specs:
+        raise ValueError(
+            "Saved config has no metric definitions under 'metrics'. "
+            "Offline evaluation requires serialized metrics written at inference time."
+        )
+
+    if not isinstance(raw_specs, (list, ListConfig)):
+        raise ValueError("Saved config field 'metrics' must be a list.")
+
+    raw_spec_items = cast(list[Any], raw_specs)
+    metric_specs: list[DictConfig] = []
+    for spec in raw_spec_items:
+        if not isinstance(spec, DictConfig):
+            raise ValueError("Saved metric definitions must be DictConfig mappings.")
+        metric_specs.append(spec)
+    return deserialize_metrics(metric_specs)
+
+
+def _find_saved_config(artifact_dir: Path) -> str | None:
+    """Check if artifact directory has a saved evaluation config."""
+    for name in ["evaluation_config.json", "evaluation_config.yaml"]:
+        if (artifact_dir / name).exists():
+            return str(artifact_dir / name)
+    return None
+
+
+def _load_evaluation_metrics(artifact_path: Path) -> list[Any]:
+    """Load evaluation metrics from config saved in artifact directory."""
+    saved_config = _find_saved_config(artifact_path)
+    if not saved_config:
+        raise FileNotFoundError(
+            "No saved evaluation config found in artifact directory. "
+            "Expected evaluation_config.json or evaluation_config.yaml."
+        )
+    return _load_from_config_file(saved_config)
+
+
+def handle_evaluate(args: Any) -> None:
+    """Execute the evaluate subcommand."""
+    console = rich.console.Console()
+    artifact_path = Path(args.artifact_dir)
+
+    if not artifact_path.exists():
+        console.print(f"[red]Error: {artifact_path} not found[/red]")
+        sys.exit(1)
+
+    # Load metrics
+    metrics = _load_evaluation_metrics(artifact_path)
+    if not metrics:
+        console.print("[yellow]No metrics found in saved evaluation config.[/yellow]")
+        sys.exit(1)
+
+    console.print(f"[bold]Loaded {len(metrics)} metric(s):[/bold]")
+    for metric in metrics:
+        metric_str = "\t" + "\n\t".join(str(metric).splitlines())
+        console.print(metric_str)
+
+    # Run evaluation
+    device = torch.device(args.device)
+    console.print(f"[bold]Starting evaluation on {device!s}...[/bold]")
+
+    results = run_offline_evaluation(
+        metrics,
+        artifact_path,
+        output_dir=args.output_dir,
+        plot=args.plot,
+        force=args.force,
+        device=device,
+        show_progress=not args.quiet,
+    )
+
+    if results:
+        console.print("\n[bold green]Evaluation Complete[/bold green]")
+        for metric_name, values in results.items():
+            console.print(f"\n{metric_name}:")
+            for key, val in values.items():
+                console.print(f"  {key:30s}: {val:.4f}")
+
+        if args.output_dir:
+            console.print(f"\n[bold]Results: {args.output_dir}[/bold]")
+
+
+def main(args: list[str]) -> None:
+    """Entry point for evaluate command."""
+    parser = setup_parser()
+    parsed_args = parser.parse_args(args)
+    handle_evaluate(parsed_args)

evaluators-2.1.3/sources/evaluators/cli/third_party.py

@@ -0,0 +1,177 @@
+"""
+CLI module for third-party benchmark export.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from evaluators.third_party.adapters.dvpq import VipDeeplabKeyMap, export_vip_deeplab
+from evaluators.third_party.adapters.pq import export_pq
+
+BENCHMARK_CHOICES: tuple[str, ...] = (
+    "pq",
+    "cityscapes_dvpq",
+    "semkitti_dvpq",
+    "semkitti_dstq",
+)
+VIP_DEEPLAB_BENCHMARKS: tuple[str, ...] = (
+    "cityscapes_dvpq",
+    "semkitti_dvpq",
+    "semkitti_dstq",
+)
+
+
+def setup_parser(prog: str = "evaluators third_party") -> argparse.ArgumentParser:
+    """Build argument parser for third-party benchmark export CLI."""
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        description=(
+            "Export offline memory artifacts for third-party benchmark scripts."
+        ),
+    )
+    subparsers = parser.add_subparsers(dest="subcommand", required=True)
+
+    export_parser = subparsers.add_parser(
+        "export",
+        help="Export artifacts for a third-party benchmark script.",
+    )
+    export_parser.add_argument(
+        "--memory",
+        type=str,
+        required=True,
+        help="Path to the offline memory artifact directory.",
+    )
+    export_parser.add_argument(
+        "--benchmark",
+        type=str,
+        required=True,
+        choices=BENCHMARK_CHOICES,
+        help="Benchmark adapter target.",
+    )
+    export_parser.add_argument(
+        "--output",
+        type=str,
+        required=True,
+        help="Output directory for exported files.",
+    )
+    export_parser.add_argument(
+        "--pred-panoptic-key",
+        type=str,
+        required=True,
+        help="Key name for predicted panoptic tensor in memory.",
+    )
+    export_parser.add_argument(
+        "--frame-index-key",
+        type=str,
+        required=True,
+        help="Key name for frame index tensor in memory.",
+    )
+    export_parser.add_argument(
+        "--target-panoptic-key",
+        type=str,
+        help="Key name for target panoptic tensor in memory (required for pq).",
+    )
+    export_parser.add_argument(
+        "--pred-depth-key",
+        type=str,
+        help="Key name for predicted depth tensor in memory (required for DVPQ/DSTQ).",
+    )
+    export_parser.add_argument(
+        "--panoptic-divisor",
+        type=int,
+        required=True,
+        help="Divisor used to encode panoptic IDs as category*divisor + instance.",
+    )
+    export_parser.add_argument(
+        "--thing-category-ids",
+        type=str,
+        required=True,
+        help="Comma-separated thing category IDs used to set COCO isthing flags.",
+    )
+    return parser
+
+
+def _require_arg(value: str | None, flag: str, benchmark: str) -> str:
+    if value:
+        return value
+    raise ValueError(f"{flag} is required for benchmark '{benchmark}'.")
+
+
+def _parse_thing_category_ids(value: str) -> set[int]:
+    raw_parts = [part.strip() for part in value.split(",")]
+    if any(part == "" for part in raw_parts):
+        raise ValueError("--thing-category-ids must be a comma-separated list of ints.")
+    ids: set[int] = set()
+    for part in raw_parts:
+        try:
+            category_id = int(part)
+        except ValueError as exc:
+            raise ValueError(
+                f"Invalid thing category id '{part}'. Expected integer values."
+            ) from exc
+        if category_id < 0:
+            raise ValueError("Thing category ids must be non-negative integers.")
+        ids.add(category_id)
+    return ids
+
+
+def handle_export(args: argparse.Namespace) -> None:
+    memory = Path(args.memory)
+    output = Path(args.output)
+    thing_category_ids = _parse_thing_category_ids(args.thing_category_ids)
+
+    if args.benchmark == "pq":
+        key_map = {
+            "pred_panoptic": args.pred_panoptic_key,
+            "target_panoptic": _require_arg(
+                args.target_panoptic_key, "--target-panoptic-key", args.benchmark
+            ),
+            "frame_index": args.frame_index_key,
+        }
+        summary = export_pq(
+            memory=memory,
+            output=output,
+            key_map=key_map,
+            panoptic_divisor=args.panoptic_divisor,
+            thing_category_ids=thing_category_ids,
+        )
+        print(
+            f"Export complete for '{args.benchmark}'. "
+            f"images={summary['images']}, "
+            f"predictions={summary['predictions']}, "
+            f"targets={summary['targets']}."
+        )
+        return
+
+    vip_key_map: VipDeeplabKeyMap = {
+        "pred_panoptic": args.pred_panoptic_key,
+        "frame_index": args.frame_index_key,
+        "pred_depth": _require_arg(
+            args.pred_depth_key, "--pred-depth-key", args.benchmark
+        ),
+    }
+    if args.benchmark not in VIP_DEEPLAB_BENCHMARKS:
+        raise ValueError(f"Unsupported ViP-DeepLab benchmark: {args.benchmark}")
+    summary = export_vip_deeplab(
+        memory=memory,
+        output=output,
+        key_map=vip_key_map,
+        format_name=args.benchmark,
+        panoptic_divisor=args.panoptic_divisor,
+    )
+    print(
+        f"Export complete for '{args.benchmark}'. "
+        f"images={summary['images']}, "
+        f"panoptic_files={summary['panoptic_files']}, "
+        f"depth_files={summary['depth_files']}."
+    )
+
+
+def main(args: list[str] | None = None) -> None:
+    """Entry point for third_party command."""
+    parser = setup_parser()
+    parsed_args = parser.parse_args(args)
+    if parsed_args.subcommand == "export":
+        handle_export(parsed_args)

{evaluators-2.1.1 → evaluators-2.1.3}/sources/evaluators/integrations/lightning.py

@@ -12,12 +12,15 @@ Exports
 
 import logging
 import shutil
+from collections.abc import Sequence
 from pathlib import Path
+from typing import Protocol
 
 from evaluators import MetricStream, run_offline_evaluation
 from evaluators.memory import (
     MemmapTemporalWriter,
     TemporalWriterProtocol,
+    save_evaluation_config,
 )
 from evaluators.utils import comms
 from lightning import Callback, LightningModule, Trainer
@@ -27,6 +30,44 @@ __all__ = ["DistributedEvaluationCallback", "DistributedInferenceWriter"]
 logger = logging.getLogger(__name__)
 
 
+class PredictEvaluationCallable(Protocol):
+    """Signature for post-prediction evaluation callables."""
+
+    def __call__(
+        self,
+        streams: Sequence[MetricStream],
+        artifact_root: str | Path,
+        *,
+        trainer: Trainer | None = None,
+        pl_module: LightningModule | None = None,
+    ) -> dict[str, dict[str, dict[str, float]]] | None:
+        """Run evaluation for persisted artifacts."""
+        ...
+
+
+def _default_evaluate_streams(
+    streams: Sequence[MetricStream],
+    artifact_root: str | Path,
+    *,
+    trainer: Trainer | None = None,
+    pl_module: LightningModule | None = None,
+) -> dict[str, dict[str, dict[str, float]]]:
+    """Default post-predict evaluator using offline evaluation."""
+    del trainer, pl_module
+    artifact_root = Path(artifact_root)
+    all_results: dict[str, dict[str, dict[str, float]]] = {}
+
+    for stream in streams:
+        stream_results = run_offline_evaluation(
+            metrics=stream.metrics,
+            artifact_dir=artifact_root / stream.name,
+            show_progress=False,
+        )
+        all_results[stream.name] = stream_results
+
+    return all_results
+
+
 class _DistributedWriterBase(Callback):
     """
     Base class for callbacks that need to bind MetricStreams to Disk Writers.
@@ -83,6 +124,11 @@ class _DistributedWriterBase(Callback):
                 **self.backend_kwargs,
             )
             stream.bind(writer)
+            save_evaluation_config(
+                output_dir=stream_dir,
+                metrics=stream.metrics,
+                metadata={"stream_name": stream.name},
+            )
             self._active_streams[stream.name] = (stream, writer)
 
     def _close_writers(self) -> None:
@@ -186,11 +232,34 @@ class DistributedInferenceWriter(_DistributedWriterBase):
     Does NOT run any metrics. Useful for generating offline artifacts.
     """
 
+    def __init__(
+        self,
+        output_dir: str | Path,
+        *,
+        backend: type[TemporalWriterProtocol] = MemmapTemporalWriter,
+        evaluate: PredictEvaluationCallable | None = _default_evaluate_streams,
+        **backend_kwargs,
+    ) -> None:
+        super().__init__(
+            output_dir=output_dir,
+            backend=backend,
+            **backend_kwargs,
+        )
+        self.evaluate = evaluate
+
     def on_predict_start(self, trainer: Trainer, pl_module: LightningModule) -> None:
         self._setup_streams(pl_module)
 
-    def on_predict_epoch_end(
-        self, trainer: Trainer, pl_module: LightningModule
-    ) -> None:
+    def on_predict_end(self, trainer: Trainer, pl_module: LightningModule) -> None:
         self._close_writers()
+
+        if self.evaluate is not None:
+            streams = [stream for stream, _ in self._active_streams.values()]
+            self.evaluate(
+                streams,
+                artifact_root=self.output_dir,
+                trainer=trainer,
+                pl_module=pl_module,
+            )
+
         self._active_streams = {}