geodispbench3d 0.2.0__py3-none-any.whl

geodispbench3d/__init__.py

@@ -0,0 +1,19 @@
+"""geodispbench3d: a generic benchmark framework for 3D displacement / optical-flow tools.
+
+The package is tool-agnostic: any tool that can be described by a
+:class:`~geodispbench3d.tool.base.ToolAdapter` can be swept, evaluated against a
+dataset, and scored with configurable metrics.
+
+Public surface is intentionally small; see the submodule docstrings for details.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "tool",
+    "dataset",
+    "metrics",
+    "sweep",
+    "suite",
+    "results",
+]

geodispbench3d/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
+
+__commit_id__ = commit_id = 'g22a498d'

geodispbench3d/analysis/__init__.py

@@ -0,0 +1,31 @@
+"""Tool-agnostic analysis: re-score cached predictions across tools.
+
+Where a suite's ``rescore`` pass is bound to one tool's run directories
+and runs phase 2 against tool-specific outputs, an analysis YAML works
+purely from the predictions cache. Predictions live in the common
+``{per_point: [...]}`` shape, so an analysis can mix iof3D and F2S3
+results in one parquet output, with the metric set as the only knob.
+"""
+
+from __future__ import annotations
+
+from .loader import (
+    AnalysisConfig,
+    PredictionFilter,
+    PredictionRef,
+    PredictionsConfig,
+    ResultsConfig,
+    load_analysis,
+)
+from .runner import AnalysisSummary, analyze
+
+__all__ = [
+    "AnalysisConfig",
+    "AnalysisSummary",
+    "PredictionFilter",
+    "PredictionRef",
+    "PredictionsConfig",
+    "ResultsConfig",
+    "analyze",
+    "load_analysis",
+]

geodispbench3d/analysis/loader.py

@@ -0,0 +1,209 @@
+"""``analysis.yaml`` schema and loader.
+
+An analysis YAML composes a dataset + a metrics file with a set of
+*cached predictions*. There is no tool reference: predictions are tool-
+agnostic by the time they reach the cache, so the analysis verb can mix
+runs from any number of tools in a single parquet output.
+
+Three ways to point at predictions, mix-and-match in any combination:
+
+  predictions:
+    - path: <abs/relative.json>          # explicit single file
+    - glob: <pattern>                    # any pattern resolved relative
+                                         # to the analysis YAML
+    - root: <dir>                        # walk the cache layout under
+      filter:                            # this root, optionally
+        tool_id: iof3d-v2                # filtering by provenance
+        dataset_id: mattertal            # segment. Each filter is
+        case: mattertal-all              # optional.
+
+Resolution returns a flat list of prediction file paths in the order
+declared (with glob results sorted lexicographically).
+"""
+
+from __future__ import annotations
+
+import glob as _glob
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from omegaconf import OmegaConf
+
+from geodispbench3d.dataset.schema import DatasetSpec, load_dataset
+from geodispbench3d.metrics.registry import MetricsConfig, load_metrics_config
+from geodispbench3d.results.predictions_cache import find_predictions
+
+
+@dataclass(frozen=True)
+class PredictionFilter:
+    """Provenance filter for a ``root:`` entry in ``predictions:``.
+
+    Each ``None`` field matches any value in that segment of the cache
+    layout (``<root>/<tool_id>/<dataset_id>/<case>/<run_hash>.json``).
+    """
+
+    tool_id: str | None = None
+    dataset_id: str | None = None
+    case: str | None = None
+
+
+@dataclass(frozen=True)
+class PredictionRef:
+    """One source of predictions to consume.
+
+    Exactly one of ``path``, ``glob``, or ``root`` is populated. The
+    loader normalises everything to an iterable of resolved Path objects
+    via :meth:`resolve`.
+    """
+
+    path: Path | None = None
+    glob: str | None = None
+    root: Path | None = None
+    filter: PredictionFilter = field(default_factory=PredictionFilter)
+
+    def resolve(self) -> list[Path]:
+        if self.path is not None:
+            return [self.path] if self.path.is_file() else []
+        if self.glob is not None:
+            return sorted(Path(p) for p in _glob.glob(self.glob, recursive=True))
+        if self.root is not None:
+            return find_predictions(
+                self.root,
+                tool_id=self.filter.tool_id,
+                dataset_id=self.filter.dataset_id,
+                case=self.filter.case,
+            )
+        return []
+
+
+@dataclass(frozen=True)
+class PredictionsConfig:
+    """The ``predictions:`` block plus an aggregate resolver."""
+
+    refs: Sequence[PredictionRef] = ()
+
+    def resolve_all(self) -> list[Path]:
+        seen: set[Path] = set()
+        ordered: list[Path] = []
+        for ref in self.refs:
+            for path in ref.resolve():
+                resolved = path.resolve()
+                if resolved in seen:
+                    continue
+                seen.add(resolved)
+                ordered.append(resolved)
+        return ordered
+
+
+@dataclass(frozen=True)
+class ResultsConfig:
+    parquet_path: Path | None = None
+
+
+@dataclass(frozen=True)
+class AnalysisConfig:
+    """Composite analysis definition with all referenced configs loaded."""
+
+    id: str
+    dataset: DatasetSpec
+    metrics: MetricsConfig
+    predictions: PredictionsConfig
+    results: ResultsConfig = field(default_factory=ResultsConfig)
+    pass_id: str | None = None
+    source_path: Path | None = None
+
+
+def load_analysis(path: str | Path) -> AnalysisConfig:
+    """Load an ``analysis.yaml`` and resolve its references."""
+
+    yaml_path = Path(path).resolve()
+    if not yaml_path.is_file():
+        raise FileNotFoundError(f"Analysis YAML not found: {yaml_path}")
+
+    raw = OmegaConf.to_container(OmegaConf.load(str(yaml_path)), resolve=True)
+    if not isinstance(raw, dict):
+        raise ValueError(f"Analysis YAML at {yaml_path} must be a mapping")
+
+    base = yaml_path.parent
+
+    dataset_ref = raw.get("dataset")
+    metrics_ref = raw.get("metrics")
+    if not (dataset_ref and metrics_ref):
+        raise ValueError(f"Analysis {yaml_path} must reference 'dataset' and 'metrics'")
+
+    dataset_spec = load_dataset(_resolve_path(dataset_ref, base))
+    metrics_cfg = load_metrics_config(_resolve_path(metrics_ref, base))
+
+    predictions_raw = raw.get("predictions") or []
+    if not isinstance(predictions_raw, list):
+        raise ValueError(f"Analysis {yaml_path}: 'predictions' must be a list")
+    refs = tuple(_load_prediction_ref(entry, base) for entry in predictions_raw)
+    if not refs:
+        raise ValueError(f"Analysis {yaml_path}: at least one prediction source required")
+
+    results_raw = raw.get("results") or {}
+    results = ResultsConfig(
+        parquet_path=_resolve_optional_path(results_raw.get("parquet_path"), base),
+    )
+
+    return AnalysisConfig(
+        id=str(raw.get("id", yaml_path.stem)),
+        dataset=dataset_spec,
+        metrics=metrics_cfg,
+        predictions=PredictionsConfig(refs=refs),
+        results=results,
+        pass_id=raw.get("pass_id"),
+        source_path=yaml_path,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Internals
+# ---------------------------------------------------------------------------
+
+
+def _load_prediction_ref(entry: Mapping[str, Any], base: Path) -> PredictionRef:
+    if "path" in entry:
+        return PredictionRef(path=_resolve_path(entry["path"], base))
+    if "glob" in entry:
+        # Resolve relative to the analysis YAML's directory.
+        pattern = str(entry["glob"])
+        if not Path(pattern).is_absolute():
+            pattern = str(base / pattern)
+        return PredictionRef(glob=pattern)
+    if "root" in entry:
+        flt = entry.get("filter") or {}
+        return PredictionRef(
+            root=_resolve_path(entry["root"], base),
+            filter=PredictionFilter(
+                tool_id=flt.get("tool_id"),
+                dataset_id=flt.get("dataset_id"),
+                case=flt.get("case"),
+            ),
+        )
+    raise ValueError(
+        f"predictions entry must declare one of 'path', 'glob', or 'root', got {dict(entry)!r}"
+    )
+
+
+def _resolve_path(value: Any, base: Path) -> Path:
+    p = Path(str(value))
+    return p if p.is_absolute() else (base / p).resolve()
+
+
+def _resolve_optional_path(value: Any, base: Path) -> Path | None:
+    if value is None:
+        return None
+    return _resolve_path(value, base)
+
+
+__all__ = [
+    "AnalysisConfig",
+    "PredictionFilter",
+    "PredictionRef",
+    "PredictionsConfig",
+    "ResultsConfig",
+    "load_analysis",
+]

geodispbench3d/analysis/runner.py

@@ -0,0 +1,205 @@
+"""Analyze runner: score cached predictions against an AnalysisConfig.
+
+Loads each prediction JSON file, picks the matching dataset case from
+the analysis config (preferring the case recorded in the prediction's
+provenance, falling back to a single-case dataset), and dispatches the
+metric registry through :func:`evaluate_trial` with the cached
+prediction supplied as ``prediction_override`` so phase 2 is skipped
+entirely.
+
+Record rows carry ``mode="analyze"`` plus the prediction's recorded
+``tool_id`` / ``dataset_id`` / ``case`` so a single parquet file can
+mix runs from multiple tools across multiple analyses without
+columns colliding.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from geodispbench3d.dataset.schema import CaseSpec, DatasetSpec
+from geodispbench3d.diagnostics import PassDiagnostics
+from geodispbench3d.metrics.registry import MetricRegistry
+from geodispbench3d.results.predictions_cache import read_prediction
+from geodispbench3d.sweep.evaluation import evaluate_trial
+from geodispbench3d.tool.base import TrialOutputs, TrialResult
+
+from .loader import AnalysisConfig
+
+
+@dataclass
+class AnalysisSummary:
+    """One-line counters returned by :func:`analyze`.
+
+    ``non_fatal_failures`` totals the swallowed fail-soft failures across the
+    pass (corrupt prediction reads, per-prediction evaluation skips), surfaced
+    as the CLI's aggregate "N non-fatal failures" line (F-08).
+
+    ``eval_failures`` (03-01) is the genuine-parser/metric subset — the
+    ``"evaluation"`` diag kind (both the raise-case skip and the inner
+    evaluation non-fatals) — which Plan 02 reads for the analyze exit-1
+    condition. ``skipped_unreadable`` stays a genuine data error and
+    ``skipped_no_case`` stays a benign skip.
+    """
+
+    total: int = 0
+    succeeded: int = 0
+    skipped_unreadable: int = 0
+    skipped_no_case: int = 0
+    rows_emitted: int = 0
+    non_fatal_failures: int = 0
+    eval_failures: int = 0
+
+
+def analyze(
+    *,
+    config: AnalysisConfig,
+    on_record_rows: Callable[[Sequence[Mapping[str, Any]]], None] | None = None,
+    logger: logging.Logger | None = None,
+) -> AnalysisSummary:
+    """Score every prediction referenced by ``config`` and emit record rows."""
+
+    log = logger or logging.getLogger("geodispbench3d.analysis")
+    summary = AnalysisSummary()
+    pass_id = config.pass_id or _utcnow_compact()
+
+    case_index: Mapping[str, CaseSpec] = {c.name: c for c in config.dataset.cases}
+    registry = MetricRegistry()
+
+    # One PassDiagnostics for the pass: corrupt prediction reads and per-
+    # prediction evaluation skips record here, surfaced on AnalysisSummary (F-08).
+    diag = PassDiagnostics()
+
+    paths = config.predictions.resolve_all()
+    log.info("analyze: %d prediction file(s) to score (pass_id=%s)", len(paths), pass_id)
+
+    for path in paths:
+        summary.total += 1
+        payload = read_prediction(path, on_non_fatal=lambda _exc: diag.add("prediction_read"))
+        if payload is None:
+            log.warning("analyze: cannot read %s, skipping", path)
+            summary.skipped_unreadable += 1
+            continue
+
+        prediction = payload.get("prediction")
+        provenance = payload.get("provenance") or {}
+        case = _resolve_case(provenance, case_index, config.dataset)
+        if case is None:
+            log.warning(
+                "analyze: cannot map prediction %s to a dataset case (provenance=%s); skipping",
+                path,
+                provenance.get("dataset"),
+            )
+            summary.skipped_no_case += 1
+            continue
+
+        record_extras = {
+            "tool_id": _provenance_id(provenance, "tool"),
+            "dataset_id": _provenance_id(provenance, "dataset") or config.dataset.id,
+            "case": case.name,
+            "trial_index": _provenance_run_hash(provenance, path),
+            "mode": "analyze",
+            "pass_id": pass_id,
+            "prediction_path": str(path),
+        }
+
+        trial_result = TrialResult(
+            outputs=TrialOutputs(run_dir=Path(provenance.get("run_dir") or path.parent)),
+            scalar_metrics={},
+            duration_seconds=0.0,
+            success=True,
+        )
+
+        try:
+            evaluation = evaluate_trial(
+                trial_result=trial_result,
+                parameters={},
+                case=case,
+                metrics=config.metrics,
+                registry=registry,
+                output_parser=None,
+                output_parser_options=None,
+                prediction_override=prediction,
+                trial_index=None,
+                record_extras=record_extras,
+                logger=log,
+            )
+        except Exception:
+            # Plugin/user callable boundary: evaluate_trial runs arbitrary
+            # metric code, so a closed exception set is inapplicable. Stay broad
+            # so one prediction's failure skips it instead of aborting the whole
+            # analyze pass (fail-soft, F-08).
+            log.exception("analyze: evaluate_trial raised for %s", path)
+            diag.add("evaluation")
+            continue
+
+        diag.add("evaluation", evaluation.non_fatal_failures)
+
+        if on_record_rows and evaluation.record_rows:
+            on_record_rows(list(evaluation.record_rows))
+            summary.rows_emitted += len(evaluation.record_rows)
+        summary.succeeded += 1
+
+    summary.non_fatal_failures = diag.non_fatal_failures
+    summary.eval_failures = diag.by_kind.get("evaluation", 0)
+    log.info(
+        "analyze done: succeeded=%d total=%d unreadable=%d no_case=%d rows=%d "
+        "non_fatal_failures=%d",
+        summary.succeeded,
+        summary.total,
+        summary.skipped_unreadable,
+        summary.skipped_no_case,
+        summary.rows_emitted,
+        summary.non_fatal_failures,
+    )
+    return summary
+
+
+# ---------------------------------------------------------------------------
+# Internals
+# ---------------------------------------------------------------------------
+
+
+def _resolve_case(
+    provenance: Mapping[str, Any],
+    case_index: Mapping[str, CaseSpec],
+    dataset: DatasetSpec,
+) -> CaseSpec | None:
+    block = provenance.get("dataset")
+    if isinstance(block, Mapping):
+        name = block.get("case")
+        if isinstance(name, str) and name in case_index:
+            return case_index[name]
+    if len(dataset.cases) == 1:
+        return dataset.cases[0]
+    return None
+
+
+def _provenance_id(provenance: Mapping[str, Any], key: str) -> str | None:
+    block = provenance.get(key)
+    if isinstance(block, Mapping):
+        value = block.get("id")
+        if isinstance(value, str):
+            return value
+    return None
+
+
+def _provenance_run_hash(provenance: Mapping[str, Any], path: Path) -> str:
+    """Best-effort identifier for the row's `trial_index` column."""
+
+    run_dir = provenance.get("run_dir")
+    if isinstance(run_dir, str) and run_dir:
+        return Path(run_dir).name
+    return path.stem
+
+
+def _utcnow_compact() -> str:
+    return datetime.now(UTC).strftime("analyze-%Y%m%dT%H%M%S")
+
+
+__all__ = ["AnalysisSummary", "analyze"]