evalrx 0.1.2__py3-none-any.whl

evalrx/eval_agent/stages/whitebox_probe_generator.py

@@ -0,0 +1,351 @@
+"""M1 tier (b), white-box — capture-then-compute probe generation.
+
+Black-box generated probes (:mod:`probe_generator`) only see the model's text
+outputs.  Mechanism hypotheses from M3 ("structural tokens act as attention
+sinks", "attention never leaves the text region", …) need **internal** evidence
+— but the loaded model cannot be shipped into a sandbox.  The split here keeps
+the M2-tier(b) safety model intact:
+
+1. **Host captures** (trusted code): one ``model.forward`` per case with
+   ``Capability.ATTENTION``; the heavy trace is immediately reduced to a small
+   standard bundle — head-averaged attention from the last query position for
+   every layer ``(n_layers, seq)`` float16 — plus the token strings and the
+   image-token mask, dumped to the sandbox workdir.
+2. **Sandboxed compute** (generated code): an LLM/CLI-written numpy script
+   reads the dump, computes a per-case scalar/boolean signal for the
+   hypothesised mechanism, and prints a strict ``PROBE_RESULT_JSON=`` line.
+3. The parsed findings become a normal :class:`~evalrx.core.result.Result`
+   whose ``per_case`` entries flow into M2's stats layer and M4's routing.
+
+Dump layout (documented verbatim in the generation prompt):
+
+    m1_whitebox_manifest.json   {"cases": [{"id", "label", "seq_len",
+                                            "n_layers", "tokens": [...]}, ...]}
+    m1_whitebox/<case_id>.npz   attn_last:        (n_layers, seq) float16
+                                image_token_mask: (seq,) bool
+
+Scope (v1): read-only statistics over the captured attention bundle.  Causal
+interventions (activation patching, attention knockout) need in-process
+execution with the model handle and are deliberately out of scope.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional
+
+import numpy as np
+
+from evalrx.agent_runtime.sandbox import ExperimentSandbox
+from evalrx.core.capability import Capability
+from evalrx.core.result import Result
+from evalrx.eval_agent.prompts.whitebox_probe_generator import (
+    _DUMP_DIR,
+    _GENERATE_PROMPT,
+    _MANIFEST,
+    _RESULT_MARKER,
+)
+
+if TYPE_CHECKING:
+    from evalrx.agent_runtime.cli_types import CliAgentConfig
+    from evalrx.core.case import CaseBatch
+    from evalrx.core.model import Model
+
+logger = logging.getLogger(__name__)
+
+
+
+@dataclass
+class GeneratedWhiteboxProbe:
+    """A validated, sandbox-executed white-box probe (re-runnable on new cases)."""
+
+    name: str
+    code: str
+    need: str = ""
+    source: str = ""
+
+
+class WhiteboxProbeGenerator:
+    """Generate and run capture-then-compute attention probes.
+
+    Args:
+        judge:       LLM for the single-pass code-writing path.
+        cli_config:  CLI coding-agent config (used before the judge when set).
+        sandbox:     Execution sandbox (fresh temp dir when ``None``).
+        timeout_sec: Wall-clock limit per sandbox run.
+        max_cases:   Cap on captured cases (one ATTENTION forward each); 0 (the default) = every case.
+    """
+
+    def __init__(
+        self,
+        judge: "Model | None" = None,
+        cli_config: "CliAgentConfig | None" = None,
+        sandbox: "ExperimentSandbox | None" = None,
+        timeout_sec: int = 60,
+        max_cases: int = 0,
+        run_logger: "Any | None" = None,
+    ) -> None:
+        self._judge = judge
+        self._cli_config = cli_config
+        self._timeout_sec = timeout_sec
+        self._max_cases = max_cases
+        self._sandbox = sandbox or ExperimentSandbox()
+        # Optional RunLogger — records each attention-probe code-writing attempt.
+        self.run_logger = run_logger
+        self._last_prompt: str = ""
+        self._last_raw: str = ""
+        self._last_raw_stream: str = ""
+        self._last_usage: dict | None = None
+
+    @property
+    def available(self) -> bool:
+        return self._judge is not None or (
+            self._cli_config is not None and self._cli_config.provider != "llm"
+        )
+
+    # ------------------------------------------------------------------
+    # Public interface
+    # ------------------------------------------------------------------
+
+    def generate(
+        self,
+        need: str,
+        model: "Model",
+        cases: "CaseBatch",
+        name: str = "custom",
+    ) -> "tuple[Result | None, GeneratedWhiteboxProbe | None]":
+        """Capture internals, write+run a probe over the dump, return (Result, probe)."""
+        if not self.available:
+            logger.debug("WhiteboxProbeGenerator: no code-writing backend configured")
+            return None, None
+        if Capability.ATTENTION not in getattr(model, "capabilities", frozenset()):
+            logger.debug("WhiteboxProbeGenerator: model lacks ATTENTION capability")
+            return None, None
+
+        n = self._capture(model, cases)
+        if n == 0:
+            logger.warning("WhiteboxProbeGenerator: no case could be captured")
+            return None, None
+
+        self._last_prompt = ""
+        self._last_raw = ""
+        self._last_raw_stream = ""
+        self._last_usage = None
+        try:
+            code, source = self._write_code(need)
+        except Exception as exc:
+            logger.warning("WhiteboxProbeGenerator: code writing failed: %s", exc)
+            self._emit_codegen(name, need, "", "", ok=False, error=f"code writing failed: {exc}")
+            return None, None
+        if not code.strip():
+            self._emit_codegen(name, need, source, "", ok=False, error="empty code produced")
+            return None, None
+
+        result = self._run_code(code, name, model, cases)
+        self._emit_codegen(
+            name, need, source, code, ok=result is not None,
+            error="" if result is not None else "sandbox produced no parseable result",
+        )
+        if result is None:
+            return None, None
+        probe = GeneratedWhiteboxProbe(name=name, code=code, need=need, source=source)
+        return result, probe
+
+    def _emit_codegen(
+        self, name: str, need: str, source: str, code: str, *, ok: bool, error: str = ""
+    ) -> None:
+        """Record one attention-probe code-writing attempt to the RunLogger."""
+        if self.run_logger is None:
+            return
+        extra = ({"cli_usage": self._last_usage}
+                 if source.startswith("cli:") and self._last_usage else None)
+        try:
+            self.run_logger.log_tool_codegen(
+                module="m1_whitebox", name=name, need=need, source=source, ok=ok,
+                code=code, prompt=self._last_prompt, raw_output=self._last_raw,
+                raw_stream=self._last_raw_stream, error=error,
+                extra=extra,
+            )
+        except Exception as exc:  # logging must never break generation
+            logger.debug("WhiteboxProbeGenerator: log_tool_codegen failed: %s", exc)
+
+    def run_cached(
+        self,
+        probe: GeneratedWhiteboxProbe,
+        model: "Model",
+        cases: "CaseBatch",
+    ) -> "Result | None":
+        """Re-capture and re-run an already-generated probe (no LLM call)."""
+        if self._capture(model, cases) == 0:
+            return None
+        return self._run_code(probe.code, probe.name, model, cases)
+
+    # ------------------------------------------------------------------
+    # Host capture (trusted)
+    # ------------------------------------------------------------------
+
+    def _capture(self, model: "Model", cases: "CaseBatch") -> int:
+        """Forward each case, reduce, and dump the attention bundle. Returns n captured."""
+        workdir = Path(self._sandbox.workdir)
+        dump_dir = workdir / _DUMP_DIR
+        dump_dir.mkdir(parents=True, exist_ok=True)
+
+        manifest: list[dict[str, Any]] = []
+        selected = list(cases)
+        if self._max_cases > 0:  # 0 = every case
+            selected = selected[: self._max_cases]
+        if getattr(self.run_logger, "preserve_full_model_io", False):
+            from evalrx.eval_agent.model_instrumentation import InstrumentedModel
+
+            model = InstrumentedModel(
+                model, self.run_logger,
+                cycle=int(getattr(self.run_logger, "current_cycle", -1)),
+                analyzer="generated_whitebox_probe_capture",
+                case_prompts={c.inputs.prompt: c.id for c in selected},
+                batch_case_ids=[c.id for c in selected],
+            )
+        for case in selected:
+            try:
+                trace = model.forward(case.inputs, capture={Capability.ATTENTION})
+                attns = trace.require(Capability.ATTENTION)
+                rows = np.stack([
+                    a.float().mean(dim=0)[-1].cpu().numpy() for a in attns
+                ]).astype(np.float16)                       # (n_layers, seq)
+                mask = trace.extras.get("image_token_mask")
+                mask_np = (
+                    mask.cpu().numpy().astype(bool)
+                    if mask is not None else np.zeros(rows.shape[1], dtype=bool)
+                )
+                tokens = list(trace.tokens)
+            except Exception as exc:  # noqa: BLE001 - skip uncapturable cases
+                logger.debug("whitebox capture failed for %s: %s", case.id, exc)
+                continue
+            finally:
+                trace = None  # free the heavy attention tensors promptly
+
+            np.savez_compressed(
+                dump_dir / f"{case.id}.npz",
+                attn_last=rows, image_token_mask=mask_np,
+            )
+            label = getattr(case, "label", None)
+            manifest.append({
+                "id": case.id,
+                "label": getattr(label, "value", None),
+                "seq_len": int(rows.shape[1]),
+                "n_layers": int(rows.shape[0]),
+                "tokens": tokens,
+            })
+
+        (workdir / _MANIFEST).write_text(
+            json.dumps({"cases": manifest}), encoding="utf-8"
+        )
+        return len(manifest)
+
+    # ------------------------------------------------------------------
+    # Code writing + sandbox run (mirrors probe_generator)
+    # ------------------------------------------------------------------
+
+    def _write_code(self, need: str) -> tuple[str, str]:
+        if self._cli_config is not None and self._cli_config.provider != "llm":
+            code = self._write_code_cli(need)
+            if code:
+                return code, f"cli:{self._cli_config.provider}"
+        prompt = self._build_prompt(need, fenced=True)
+        self._last_prompt = prompt
+        raw = self._judge.generate(prompt)  # type: ignore[union-attr]
+        self._last_raw = str(raw)
+        return _extract_code(str(raw)), "llm"
+
+    def _write_code_cli(self, need: str) -> str:
+        from evalrx.agent_runtime.codegen import CodegenRunner
+
+        prompt = self._build_prompt(need, fenced=False)
+        self._last_prompt = prompt
+        result = CodegenRunner(self._cli_config).write_code(  # type: ignore[arg-type]
+            prompt,
+            workdir=Path(self._sandbox.workdir),
+            timeout_sec=self._timeout_sec,
+            preferred_filenames=("probe.py",),
+        )
+        self._last_raw = result.raw_output
+        self._last_raw_stream = ""
+        if result.raw_stream_path:
+            try:
+                self._last_raw_stream = (
+                    Path(self._sandbox.workdir) / result.raw_stream_path
+                ).read_text(encoding="utf-8")
+            except OSError:
+                pass
+        self._last_usage = result.usage
+        return result.code
+
+    def _build_prompt(self, need: str, *, fenced: bool) -> str:
+        return _GENERATE_PROMPT.format(
+            need=need.strip() or "Probe the attention pattern behind the failures.",
+            manifest=_MANIFEST,
+            dump_dir=_DUMP_DIR,
+            marker=_RESULT_MARKER,
+            fences_hint=" inside a ```python code block" if fenced else
+                        ", written to a file named probe.py",
+        )
+
+    def _run_code(
+        self, code: str, name: str, model: "Model", cases: "CaseBatch",
+    ) -> "Result | None":
+        sandbox_result = self._sandbox.run(code, timeout_sec=self._timeout_sec)
+        if not sandbox_result.ok:
+            logger.warning(
+                "WhiteboxProbeGenerator: sandbox run failed (rc=%s): %s",
+                sandbox_result.returncode, (sandbox_result.stderr or "").strip()[:200],
+            )
+            return None
+        return _parse_result(sandbox_result.stdout, name, model, cases)
+
+
+# ---------------------------------------------------------------------------
+# Module helpers (shared shape with probe_generator)
+# ---------------------------------------------------------------------------
+
+def _extract_code(raw: str) -> str:
+    cleaned = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL)
+    fence = re.search(r"```(?:python)?\s*\n(.*?)```", cleaned, flags=re.DOTALL)
+    if fence:
+        return fence.group(1).strip()
+    return cleaned.strip()
+
+
+def _parse_result(
+    stdout: str, name: str, model: "Model", cases: "CaseBatch",
+) -> "Result | None":
+    marker_line: Optional[str] = None
+    for line in stdout.splitlines():
+        s = line.strip()
+        if s.startswith(_RESULT_MARKER):
+            marker_line = s[len(_RESULT_MARKER):]
+    if marker_line is None:
+        logger.warning("WhiteboxProbeGenerator: no PROBE_RESULT_JSON line in output")
+        return None
+    try:
+        data = json.loads(marker_line)
+    except json.JSONDecodeError as exc:
+        logger.warning("WhiteboxProbeGenerator: unparseable PROBE_RESULT_JSON: %s", exc)
+        return None
+
+    findings: dict[str, Any] = {}
+    if isinstance(data.get("findings"), dict):
+        findings.update(data["findings"])
+    per_case = data.get("per_case")
+    if isinstance(per_case, list):
+        findings["per_case"] = [e for e in per_case if isinstance(e, dict)]
+
+    return Result(
+        analyzer=f"generated_wb:{name}",
+        model=repr(model),
+        cases=cases,
+        findings=findings,
+        metadata={"generated": True, "whitebox": True, "probe": name},
+    )

evalrx/eval_agent/store.py

@@ -0,0 +1,231 @@
+"""Store — the persistent memory that makes self-evolution possible.
+
+Without memory, an agent re-explores the same ground forever. The store
+accumulates the corpus the agent evolves over: cases it has constructed,
+results it has produced, and hypotheses it has tested. The loop writes to it
+each cycle and reads from it to decide what to try next.
+
+This module defines the interface, an in-memory reference, and a JSONL-backed
+persistent implementation (JsonlStore).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from evalrx.core.case import FailureCase
+    from evalrx.core.result import Result
+    from evalrx.eval_agent.hypothesis import Hypothesis
+
+logger = logging.getLogger(__name__)
+
+
+class Store(ABC):
+    """Persistent memory interface for the self-evolving loop."""
+
+    @abstractmethod
+    def add_case(self, case: "FailureCase") -> None: ...
+
+    @abstractmethod
+    def add_result(self, result: "Result") -> None: ...
+
+    @abstractmethod
+    def add_hypothesis(self, hypothesis: "Hypothesis") -> None: ...
+
+    @abstractmethod
+    def query(self, **filters: Any) -> list[Any]:
+        """Retrieve stored items matching *filters* (tags, label, status, …)."""
+
+    @abstractmethod
+    def summarize(self) -> dict[str, Any]:
+        """Aggregate view the agent reads to spot gaps and patterns."""
+
+
+class InMemoryStore(Store):
+    """Minimal non-persistent reference store (Stage-2 will add durability)."""
+
+    def __init__(self) -> None:
+        self.cases: list[FailureCase] = []
+        self.results: list[Result] = []
+        self.hypotheses: list[Hypothesis] = []
+
+    def add_case(self, case: "FailureCase") -> None:
+        self.cases.append(case)
+
+    def add_result(self, result: "Result") -> None:
+        self.results.append(result)
+
+    def add_hypothesis(self, hypothesis: "Hypothesis") -> None:
+        self.hypotheses.append(hypothesis)
+
+    def query(self, kind: str | None = None, **filters: Any) -> list[Any]:
+        """Retrieve stored items. ``kind`` in {cases, results, hypotheses}; filters:
+        ``label`` / ``tags`` (cases), ``status`` (hypotheses), ``analyzer`` (results)."""
+        out: list[Any] = []
+        if kind in (None, "cases"):
+            for c in self.cases:
+                if filters.get("label") is not None and getattr(c, "label", None) != filters["label"]:
+                    continue
+                tags = filters.get("tags")
+                if tags is not None and not set(tags).issubset(getattr(c, "tags", set())):
+                    continue
+                out.append(c)
+        if kind in (None, "results"):
+            for r in self.results:
+                if filters.get("analyzer") is not None and getattr(r, "analyzer", None) != filters["analyzer"]:
+                    continue
+                out.append(r)
+        if kind in (None, "hypotheses"):
+            for h in self.hypotheses:
+                if filters.get("status") is not None and getattr(h, "status", None) != filters["status"]:
+                    continue
+                out.append(h)
+        return out
+
+    def summarize(self) -> dict[str, Any]:
+        return {
+            "n_cases": len(self.cases),
+            "n_results": len(self.results),
+            "n_hypotheses": len(self.hypotheses),
+        }
+
+
+class JsonlStore(Store):
+    """Persistent JSONL-backed store.
+
+    Files written under *store_dir*:
+      - ``hypotheses.jsonl`` — one serialised :class:`Hypothesis` per line
+      - ``results.jsonl``    — one scalar-only result summary per line
+      - ``cases.jsonl``      — one case summary per line
+
+    Hypotheses survive process restart; cases and results store lightweight
+    summaries only (no numpy arrays or heavy objects).
+
+    Thread safety: appends use ``O_APPEND`` which is atomic on Linux for
+    small writes; reads reload from disk each call.
+    """
+
+    def __init__(self, store_dir: Path | str) -> None:
+        self._dir = Path(store_dir)
+        self._dir.mkdir(parents=True, exist_ok=True)
+        self._hyp_path = self._dir / "hypotheses.jsonl"
+        self._res_path = self._dir / "results.jsonl"
+        self._case_path = self._dir / "cases.jsonl"
+
+    # ------------------------------------------------------------------
+    # Store interface
+    # ------------------------------------------------------------------
+
+    def add_case(self, case: "FailureCase") -> None:
+        record: dict[str, Any] = {
+            "id": getattr(case, "id", ""),
+            "label": getattr(case, "label", None),
+            "tags": list(getattr(case, "tags", [])),
+            "prompt_snippet": str(getattr(getattr(case, "inputs", None), "prompt", ""))[:200],
+        }
+        self._append(self._case_path, record)
+
+    def add_result(self, result: "Result") -> None:
+        findings = getattr(result, "findings", {}) or {}
+        scalar_findings: dict[str, Any] = {}
+        for k, v in findings.items():
+            try:
+                scalar_findings[k] = float(v) if hasattr(v, "__float__") else str(v)
+            except Exception:
+                scalar_findings[k] = str(v)
+        record: dict[str, Any] = {
+            "analyzer": getattr(result, "analyzer", ""),
+            "findings": scalar_findings,
+            "metadata": {
+                k: str(v) for k, v in (getattr(result, "metadata", {}) or {}).items()
+                if isinstance(k, str)
+            },
+        }
+        self._append(self._res_path, record)
+
+    def add_hypothesis(self, hypothesis: "Hypothesis") -> None:
+        from evalrx.eval_agent.hypothesis import hypothesis_to_dict
+        self._append(self._hyp_path, hypothesis_to_dict(hypothesis))
+
+    def query(self, kind: str | None = None, **filters: Any) -> list[Any]:
+        """Retrieve stored items matching *filters*.
+
+        Returns reconstructed :class:`Hypothesis` objects for ``kind="hypotheses"``;
+        raw dicts for cases and results.
+        """
+        out: list[Any] = []
+
+        if kind in (None, "cases"):
+            for rec in self._load(self._case_path):
+                if filters.get("label") is not None and rec.get("label") != filters["label"]:
+                    continue
+                tags = filters.get("tags")
+                if tags is not None and not set(tags).issubset(set(rec.get("tags", []))):
+                    continue
+                out.append(rec)
+
+        if kind in (None, "results"):
+            for rec in self._load(self._res_path):
+                if filters.get("analyzer") is not None and rec.get("analyzer") != filters["analyzer"]:
+                    continue
+                out.append(rec)
+
+        if kind in (None, "hypotheses"):
+            from evalrx.eval_agent.hypothesis import hypothesis_from_dict
+            for rec in self._load(self._hyp_path):
+                h = hypothesis_from_dict(rec)
+                if filters.get("status") is not None:
+                    wanted = filters["status"]
+                    # Accept both enum and string comparisons
+                    h_status = h.status.value if h.status else None
+                    if wanted != h.status and wanted != h_status:
+                        continue
+                out.append(h)
+
+        return out
+
+    def summarize(self) -> dict[str, Any]:
+        n_hyp = sum(1 for _ in self._load(self._hyp_path))
+        n_res = sum(1 for _ in self._load(self._res_path))
+        n_cas = sum(1 for _ in self._load(self._case_path))
+        return {
+            "n_cases": n_cas,
+            "n_results": n_res,
+            "n_hypotheses": n_hyp,
+        }
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _append(path: Path, record: dict[str, Any]) -> None:
+        try:
+            with path.open("a", encoding="utf-8") as f:
+                f.write(json.dumps(record, default=str) + "\n")
+        except OSError as exc:
+            logger.warning("JsonlStore: failed to append to %s: %s", path, exc)
+
+    @staticmethod
+    def _load(path: Path) -> list[dict[str, Any]]:
+        if not path.exists():
+            return []
+        records: list[dict[str, Any]] = []
+        try:
+            with path.open(encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        records.append(json.loads(line))
+                    except json.JSONDecodeError:
+                        logger.debug("JsonlStore: skipping malformed line in %s", path)
+        except OSError as exc:
+            logger.warning("JsonlStore: failed to read %s: %s", path, exc)
+        return records

evalrx/logging_utils.py

@@ -0,0 +1,112 @@
+"""Console logging convenience for evalrx' existing per-module loggers.
+
+Every submodule follows the standard library idiom
+``logger = logging.getLogger(__name__)`` and already narrates real events —
+M1-M5 stage transitions, capability negotiation, paper-method dispatch
+(``vcd``/``icd``/...), fix-candidate generation and validation, audio decode
+fallbacks.  None of it is visible by default: Python's logging module only
+invokes an implicit "handler of last resort" for ``WARNING``-and-above
+records when NO handler exists anywhere in a logger's propagation chain, and
+even that prints unformatted straight to stderr.  ``.info()``/``.debug()``
+calls — most of the narration — never appear at all without a handler.
+
+A **library** must never call ``logging.basicConfig()`` itself: that
+configures the ROOT logger process-wide and would clobber a host
+application's own logging setup the moment ``import evalrx`` runs. The
+standard, documented pattern (see the stdlib logging HOWTO's "Configuring
+Logging for a Library" section) is instead:
+
+  * attach a :class:`logging.NullHandler` to the package's top-level logger
+    (done in :mod:`evalrx` on import) so the library is silent — no
+    "No handlers could be found" noise, no surprise raw stderr output — until
+    a caller opts in;
+  * expose an explicit, opt-in helper for callers who *do* want to see it.
+    That's this module.
+
+Usage::
+
+    import evalrx
+    evalrx.enable_console_logging()          # INFO and above, readable
+    model = evalrx.load("qwen2.5-7b-instruct")
+    ...                                            # now narrated to stdout
+
+    evalrx.enable_console_logging(level=logging.DEBUG)  # noisier
+    evalrx.disable_console_logging()                     # back to quiet
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from typing import IO, Optional
+
+# Every module's `logging.getLogger(__name__)` (e.g. "evalrx.eval_agent.loop")
+# is a child of this in the dot-hierarchy, so one handler here reaches all of
+# them via normal propagation -- except RunLogger's own "evalrx.run.<dir>"
+# tree, which sets propagate=False by design to keep its structured JSONL
+# event stream a separate channel from this ad-hoc narration.
+TOP_LEVEL_LOGGER_NAME = "evalrx"
+
+# Marks a handler as "ours" so enable_console_logging() is idempotent (calling
+# it twice reconfigures the level instead of stacking a second handler) and
+# disable_console_logging() only ever removes handlers it installed itself --
+# never a caller's own handler on the same logger.
+_MARKER_ATTR = "_evalrx_console_handler"
+
+
+class _ConsoleFormatter(logging.Formatter):
+    """Compact single-line format: time, level, short module name, message."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        record.short_name = record.name.rsplit(".", 1)[-1]
+        return super().format(record)
+
+
+def enable_console_logging(
+    level: int = logging.INFO, *, stream: Optional[IO[str]] = None
+) -> logging.Handler:
+    """Attach a readable stdout handler to evalrx' logger tree.
+
+    Idempotent: a second call reconfigures the level of the existing handler
+    rather than attaching a duplicate (no doubled lines).
+
+    Args:
+        level:  Minimum level to show. ``logging.INFO`` (default) surfaces the
+                stage-by-stage narration most callers want; ``logging.DEBUG``
+                is noisier (per-candidate detail, low-level capability checks).
+        stream: Defaults to ``sys.stdout``.
+
+    Returns:
+        The attached :class:`logging.Handler`, in case a caller wants to
+        remove or further customise it directly.
+    """
+    top_logger = logging.getLogger(TOP_LEVEL_LOGGER_NAME)
+    for handler in top_logger.handlers:
+        if getattr(handler, _MARKER_ATTR, False):
+            handler.setLevel(level)
+            top_logger.setLevel(level)
+            return handler
+
+    handler = logging.StreamHandler(stream or sys.stdout)
+    handler.setFormatter(
+        _ConsoleFormatter(
+            fmt="%(asctime)s %(levelname)-7s %(short_name)s: %(message)s",
+            datefmt="%H:%M:%S",
+        )
+    )
+    setattr(handler, _MARKER_ATTR, True)
+    top_logger.addHandler(handler)
+    top_logger.setLevel(level)
+    return handler
+
+
+def disable_console_logging() -> None:
+    """Remove the handler(s) installed by :func:`enable_console_logging`.
+
+    A no-op if none was ever attached. Never touches handlers a caller added
+    themselves directly to ``logging.getLogger("evalrx")`` or below.
+    """
+    top_logger = logging.getLogger(TOP_LEVEL_LOGGER_NAME)
+    for handler in list(top_logger.handlers):
+        if getattr(handler, _MARKER_ATTR, False):
+            top_logger.removeHandler(handler)