eval-toolkit 0.27.1__py3-none-any.whl

eval_toolkit/protocols.py

@@ -0,0 +1,98 @@
+"""Lightweight public Protocols with minimal dependency surface.
+
+These contracts are intentionally kept out of the heavier implementation
+modules so consumers can type adapters without importing pandas, matplotlib,
+or filesystem-oriented helpers.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+import numpy as np
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+__all__ = [
+    "EvalSliceLike",
+    "PredictionReader",
+    "Scorer",
+    "SliceAwareScorer",
+    "Versioned",
+]
+
+
+@runtime_checkable
+class Scorer(Protocol):
+    """Anything exposing ``predict_proba(X) -> np.ndarray of P(positive)``.
+
+    Accepts ``list[str]``, ``np.ndarray``, or ``pd.Series`` of features.
+    Pandas is imported under ``TYPE_CHECKING`` only, so this Protocol
+    has no runtime pandas dependency.
+    """
+
+    def predict_proba(  # pragma: no cover
+        self, X: Sequence[str] | np.ndarray | pd.Series
+    ) -> np.ndarray:
+        """Return one P(positive) score per input row."""
+        ...
+
+
+@runtime_checkable
+class SliceAwareScorer(Scorer, Protocol):
+    """Optional scorer contract for cost-controlled slice skipping."""
+
+    def should_score_slice(self, slice_name: str) -> bool:  # pragma: no cover
+        """Return whether this scorer should run on the named slice."""
+        ...
+
+
+@runtime_checkable
+class Versioned(Protocol):
+    """Anything exposing a stable version string."""
+
+    @property
+    def version(self) -> str:  # pragma: no cover
+        """Stable version string for this implementation."""
+        ...
+
+
+@runtime_checkable
+class EvalSliceLike(Protocol):
+    """Pandas-free slice surface needed by evaluation contracts."""
+
+    @property
+    def name(self) -> str:  # pragma: no cover
+        """Stable slice identifier."""
+        ...
+
+    @property
+    def y_true(self) -> np.ndarray:  # pragma: no cover
+        """Binary labels as a 1-D array."""
+        ...
+
+    @property
+    def features(self) -> Sequence[str]:  # pragma: no cover
+        """Feature values passed to a scorer."""
+        ...
+
+    @property
+    def strata(self) -> np.ndarray | None:  # pragma: no cover
+        """Optional stratum labels for slice-aware reporting."""
+        ...
+
+
+@runtime_checkable
+class PredictionReader(Protocol):
+    """Reads manifest-referenced prediction artifacts into column arrays."""
+
+    def read_predictions(  # pragma: no cover
+        self,
+        uri: str,
+        *,
+        columns: Mapping[str, str],
+    ) -> Mapping[str, Sequence[object]]:
+        """Return a column-oriented table for the requested artifact URI."""
+        ...

eval_toolkit/provenance.py

@@ -0,0 +1,255 @@
+"""Provenance helpers: file hashing, run-directory layout, optional git SHA, figure metadata.
+
+All filesystem-touching functions are isolated here so library callers can
+import :mod:`eval_toolkit.metrics` and :mod:`eval_toolkit.bootstrap` without
+pulling in any IO surface.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import subprocess
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+__all__ = [
+    "FileHash",
+    "FileHashMissing",
+    "capture_git_sha",
+    "compute_file_hash",
+    "figure_metadata",
+    "file_sha256",
+    "make_run_dir",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class FileHash:
+    """Sentinel-style hit for a successful file hash.
+
+    Returned (alongside :class:`FileHashMissing`) from
+    :func:`compute_file_hash` so callers can pattern-match on the outcome
+    rather than testing for a magic ``None`` (F5.1).
+    """
+
+    sha256: str
+
+    def __post_init__(self) -> None:
+        """Validate the hex-digest shape."""
+        if not isinstance(self.sha256, str) or len(self.sha256) != 64:
+            raise ValueError(
+                f"FileHash.sha256 must be a 64-character hex digest, got {self.sha256!r}"
+            )
+
+
+@dataclass(frozen=True, slots=True)
+class FileHashMissing:
+    """Sentinel-style miss for a file hash that could not be computed.
+
+    Carries the reason ``"missing"`` (path does not exist) or
+    ``"not_a_file"`` (path exists but is not a regular file). Pair with
+    :class:`FileHash` for exhaustive pattern matching.
+    """
+
+    reason: str
+    path: str
+
+
+def compute_file_hash(path: Path | str) -> FileHash | FileHashMissing:
+    """SHA-256 hex digest of an existing file (sentinel-typed).
+
+    Replaces the v1 :func:`file_sha256` ``str | None`` shape with an
+    explicit union (F5.1). Callers can match on the union members
+    instead of testing the return for ``None``:
+
+    .. code-block:: python
+
+        match compute_file_hash(path):
+            case FileHash(sha256=h):
+                record["digest"] = f"sha256:{h}"
+            case FileHashMissing(reason="missing"):
+                ...
+
+    :func:`file_sha256` remains available as a backward-compatible thin
+    wrapper that flattens to ``str | None``.
+
+    Examples
+    --------
+    >>> import tempfile
+    >>> from pathlib import Path
+    >>> with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f:
+    ...     _ = f.write("hello\\n")
+    ...     p = Path(f.name)
+    >>> result = compute_file_hash(p)
+    >>> isinstance(result, FileHash) and len(result.sha256) == 64
+    True
+    >>> compute_file_hash("/no/such/file")
+    FileHashMissing(reason='missing', path='/no/such/file')
+    """
+    p = Path(path)
+    if not p.exists():
+        return FileHashMissing(reason="missing", path=str(path))
+    if not p.is_file():
+        return FileHashMissing(reason="not_a_file", path=str(path))
+    h = hashlib.sha256()
+    with p.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            h.update(chunk)
+    return FileHash(sha256=h.hexdigest())
+
+
+def file_sha256(path: Path | str, *, strict: bool = False) -> str | None:
+    """SHA-256 hex digest of an existing file.
+
+    Default behavior is permissive: returns ``None`` if the path does not
+    exist or is not a regular file. Pass ``strict=True`` to raise
+    :class:`FileNotFoundError` instead — useful when the caller's invariants
+    require the digest to exist.
+
+    .. note::
+       New code should prefer :func:`compute_file_hash`, which returns a
+       :class:`FileHash` / :class:`FileHashMissing` union and avoids the
+       ``None`` ambiguity (F5.1). This helper is retained for backward
+       compatibility.
+
+    Parameters
+    ----------
+    path : pathlib.Path or str
+    strict : bool, optional
+        If ``True``, raise :class:`FileNotFoundError` when ``path`` is missing
+        or not a regular file. Default ``False`` (return ``None``).
+
+    Returns
+    -------
+    str or None
+        64-character hex digest. ``None`` only when ``strict=False`` and the
+        path is absent / not a regular file.
+
+    Raises
+    ------
+    FileNotFoundError
+        Only when ``strict=True`` and ``path`` is missing or not a regular file.
+
+    Examples
+    --------
+    >>> import tempfile
+    >>> from pathlib import Path
+    >>> with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f:
+    ...     _ = f.write("hello\\n")
+    ...     p = Path(f.name)
+    >>> digest = file_sha256(p)
+    >>> len(digest)
+    64
+    >>> file_sha256("/no/such/file") is None
+    True
+    >>> try:
+    ...     file_sha256("/no/such/file", strict=True)
+    ... except FileNotFoundError:
+    ...     print("raised")
+    raised
+    """
+    result = compute_file_hash(path)
+    if isinstance(result, FileHash):
+        return result.sha256
+    if strict:
+        raise FileNotFoundError(f"file_sha256: path missing or not a regular file: {result.path}")
+    return None
+
+
+def make_run_dir(base: Path | str, prefix: str = "run") -> Path:
+    """Create and return a timestamped run directory.
+
+    Format: ``{base}/{prefix}_{YYYY-MM-DD_HHMMSS}``.
+
+    Parameters
+    ----------
+    base : pathlib.Path or str
+        Parent directory for run dirs (e.g., ``evals/``).
+    prefix : str, optional
+        Prefix before the timestamp. Default ``"run"``.
+
+    Returns
+    -------
+    pathlib.Path
+        The newly-created (or pre-existing) run directory.
+    """
+    ts = time.strftime("%Y-%m-%d_%H%M%S")
+    run_dir = Path(base) / f"{prefix}_{ts}"
+    run_dir.mkdir(parents=True, exist_ok=True)
+    return run_dir
+
+
+def capture_git_sha(repo_root: Path | str | None = None) -> str | None:
+    """Current HEAD full SHA, or ``None`` if not in a git repo.
+
+    Optional and injectable: callers that want a specific SHA (e.g., from CI
+    metadata) should pass it directly to whichever function consumes it
+    rather than calling this helper.
+
+    Parameters
+    ----------
+    repo_root : pathlib.Path or str or None, optional
+        Working directory for ``git``. ``None`` uses the current process cwd.
+
+    Returns
+    -------
+    str or None
+        Full SHA hex string, or ``None`` if ``git`` is unavailable or the
+        directory is not inside a repository.
+    """
+    cwd = Path(repo_root) if repo_root is not None else None
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=cwd,
+        )
+        return result.stdout.strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return None
+
+
+def figure_metadata(provenance: dict[str, str], dpi: int = 300) -> dict[str, str]:
+    """Build a JSON-serializable provenance dict for ``save_figure``.
+
+    Combines caller-supplied keys with auto-fields ``timestamp_utc``
+    (ISO-8601), ``matplotlib_version``, and ``figure_dpi``. Matches the
+    schema produced inside :func:`eval_toolkit.plotting.save_figure`.
+
+    Parameters
+    ----------
+    provenance : dict[str, str]
+        Caller-supplied provenance keys (e.g., ``git_sha``, ``run_id``).
+    dpi : int, optional
+        DPI to record. Default 300.
+
+    Returns
+    -------
+    dict[str, str]
+        Combined provenance dict.
+
+    Examples
+    --------
+    >>> meta = figure_metadata({"run_id": "test-001"}, dpi=150)
+    >>> sorted(meta.keys())
+    ['figure_dpi', 'matplotlib_version', 'run_id', 'timestamp_utc']
+    >>> meta["figure_dpi"]
+    '150'
+    """
+    try:
+        import matplotlib as mpl  # noqa: PLC0415
+
+        matplotlib_version = str(mpl.__version__)
+    except ImportError:
+        matplotlib_version = "unavailable"
+
+    return {
+        **provenance,
+        "timestamp_utc": datetime.now(UTC).isoformat(timespec="seconds"),
+        "matplotlib_version": matplotlib_version,
+        "figure_dpi": str(dpi),
+    }

eval_toolkit/schemas/manifest.v1.json

@@ -0,0 +1,155 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://eval-toolkit/schemas/manifest.v1.json",
+  "version": "1",
+  "title": "eval-toolkit RunManifest",
+  "description": "Schema for run_dir/manifest.json. Produced by eval_toolkit.manifest.write_manifest. NeurIPS Reproducibility Checklist-aligned.",
+  "type": "object",
+  "required": ["schema_version", "run_id", "code_versions", "env"],
+  "properties": {
+    "schema_version": {
+      "const": "v1"
+    },
+    "run_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "git_sha": {
+      "type": ["string", "null"]
+    },
+    "dirty_flag": {
+      "type": "boolean",
+      "description": "True iff the working tree had uncommitted changes at run time (NeurIPS clean-replay concern)."
+    },
+    "code_versions": {
+      "type": "object",
+      "description": "{package_name: version_string} — at minimum eval_toolkit.",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "seeds": {
+      "type": "object",
+      "description": "{source: seed} for global / bootstrap / torch / dataloader RNGs.",
+      "additionalProperties": {
+        "type": "integer"
+      }
+    },
+    "data_hashes": {
+      "type": "object",
+      "description": "{logical_name: 'sha256:...'} for input artifacts.",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "config_hash": {
+      "type": "string",
+      "description": "SHA-256 over the canonical-JSON encoding of the eval config."
+    },
+    "env": {
+      "type": "object",
+      "description": "Environment fingerprint (python, platform, key dep versions).",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "gpu_info": {
+      "type": "object",
+      "description": "{name, count, memory_gb} from nvidia-smi (empty dict if unavailable).",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "cuda_version": {
+      "type": ["string", "null"]
+    },
+    "wall_clock_seconds": {
+      "type": ["number", "null"]
+    },
+    "versioned_objects": {
+      "type": "object",
+      "description": "{name: version} for any Tier-2 implementation exposing a `version` attribute.",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "leakage_report": {
+      "type": ["object", "null"],
+      "description": "Serialized LeakageReport (or null if no leakage checks ran).",
+      "properties": {
+        "findings": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["check_name", "severity", "drop_indices", "evidence", "message", "n_affected"],
+            "properties": {
+              "check_name": {"type": "string"},
+              "severity": {"enum": ["error", "warning", "info"]},
+              "drop_indices": {"type": "object"},
+              "evidence": {"type": "object"},
+              "message": {"type": "string"},
+              "n_affected": {"type": "integer", "minimum": 0}
+            }
+          }
+        }
+      }
+    },
+    "source_roles": {
+      "type": "array",
+      "description": "Optional generic source-role metadata for data-quality and evidence audits. Recommended roles: train, calibration, locked_eval, external_diagnostic, excluded; custom roles are allowed.",
+      "items": {
+        "type": "object",
+        "required": ["source", "role"],
+        "properties": {
+          "source": {"type": "string", "minLength": 1},
+          "role": {"type": "string", "minLength": 1},
+          "n_rows": {"type": "integer", "minimum": 0},
+          "notes": {"type": "string"},
+          "metadata": {"type": "object"}
+        },
+        "additionalProperties": true
+      }
+    },
+    "guardrails": {
+      "type": "array",
+      "description": "Optional predeclared guardrails for claim/evidence discipline.",
+      "items": {"type": "string", "minLength": 1}
+    },
+    "prediction_artifacts": {
+      "type": "array",
+      "description": "Optional references to retained prediction artifacts. The toolkit records artifact metadata and column mappings without mandating a storage format.",
+      "items": {
+        "type": "object",
+        "required": ["uri", "media_type", "columns"],
+        "properties": {
+          "uri": {"type": "string", "minLength": 1},
+          "media_type": {"type": "string", "minLength": 1},
+          "sha256": {"type": "string"},
+          "n_rows": {"type": "integer", "minimum": 0},
+          "role": {"type": "string", "minLength": 1},
+          "metadata": {"type": "object"},
+          "columns": {
+            "type": "object",
+            "required": ["label", "score"],
+            "properties": {
+              "row_id": {"type": "string", "minLength": 1},
+              "content_hash": {"type": "string", "minLength": 1},
+              "label": {"type": "string", "minLength": 1},
+              "score": {"type": "string", "minLength": 1},
+              "scorer": {"type": "string", "minLength": 1},
+              "slice": {"type": "string", "minLength": 1},
+              "text": {"type": "string", "minLength": 1},
+              "provenance": {
+                "type": "object",
+                "additionalProperties": {"type": "string"}
+              }
+            },
+            "additionalProperties": true
+          }
+        },
+        "additionalProperties": true
+      }
+    }
+  },
+  "additionalProperties": true
+}