convergent-cli 0.1.0__py3-none-any.whl

convergent_cli/calibration_frontier.py

@@ -0,0 +1,499 @@
+"""Rank many calibration treatments against one baseline on cost and quality.
+
+Each candidate, and the noise-floor treatment when one is named, is read one
+at a time through the existing compare route. Every figure comes from that
+response: the pooled (or one batch's) micro TPR and TNR deltas with their
+paired intervals, and a cost with its basis. The cost is the whole run's
+production-equivalent cost per trace, or with ``--batch`` that batch's recorded
+call cost when compare reports one. When one arm grades a strict subset of the
+other's batches, compare's production-equivalent pair covers only the shared
+batches; that row is labelled ``batch subset`` and is not ranked. Only rows
+whose cost has the baseline's scope are ranked against each other.
+Nothing here reads trace ids, labels or per-trace data, and nothing decides a
+swap.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import urllib.parse
+from collections.abc import Callable, Mapping, Sequence
+from typing import Any, TypeGuard
+
+from convergent_cli.calibration_runner import require_treatment
+from convergent_cli.command_tree import CommandCatalog, CommandInfo, Effect, Plane
+from convergent_cli.errors import ControlRefusedError, ExitCode, NetworkError
+from convergent_cli.output import emit_json
+
+Read = Callable[[str, Mapping[str, str] | None], Mapping[str, object]]
+Row = dict[str, Any]
+
+_TREATMENTS_PATH = "/v1/staff/calibration/treatments"
+_PLACED_COST_BASES = ("exact", "upper bound")
+NOT_MEASURED = "not measured"
+_DESCRIPTION = (
+    "Read compare for each candidate against the baseline, then print one row per "
+    "treatment with its TPR and TNR deltas, its cost, whether both deltas sit inside the "
+    "noise-floor intervals, and whether it is on the cost and quality frontier. TPR and "
+    "TNR count equally, and a row with unknown cost is never on the frontier. The cost is"
+    " the whole run's production-equivalent cost per trace (cost_scope=whole run). With "
+    "--batch it is that batch's recorded call cost (cost_scope=batch) when compare "
+    "reports one. A candidate that grades a different batch set than the baseline has a "
+    "cost over the shared batches only (cost_scope=batch subset); compare it with "
+    "--batch. Only rows whose cost_scope matches the baseline's are ranked. This ranks "
+    "treatments. It does not decide swaps: the evaluator swap bar (#8742) governs "
+    "replacement, and production results are not read until it is signed off."
+)
+
+
+def register(
+    commands: Any,
+    catalog: CommandCatalog,
+    *,
+    read: Read,
+    warn: Callable[[Mapping[str, object]], None],
+) -> None:
+    summary = "Rank candidate treatments against one baseline on cost and quality."
+    parser = commands.add_parser("frontier", help=summary, description=_DESCRIPTION)
+    parser.add_argument("candidates", nargs="+", help="Candidate treatment names.")
+    parser.add_argument("--baseline", required=True, help="Baseline treatment name.")
+    parser.add_argument(
+        "--noise-floor",
+        help="A rerun of the baseline. Its intervals decide within_noise.",
+    )
+    parser.add_argument("--batch", help="Read deltas from this batch member only.")
+    parser.add_argument(
+        "--basis",
+        choices=("post_audit", "pre_audit"),
+        default="post_audit",
+        help="Count each comparison from the stored or the pre-audit verdict.",
+    )
+    parser.add_argument(
+        "--format",
+        choices=("text", "json"),
+        default="text",
+        help="Print the rows as text or as a JSON array.",
+    )
+    catalog.bind(
+        parser=parser,
+        handler=lambda args: _frontier(args, read, warn),
+        info=CommandInfo(
+            path=("calibration", "frontier"),
+            description=summary,
+            plane=Plane.CONTROL,
+            requires_session=True,
+            effect=Effect.READ,
+        ),
+    )
+
+
+def _frontier(
+    args: argparse.Namespace,
+    read: Read,
+    warn: Callable[[Mapping[str, object]], None],
+) -> int:
+    baseline = require_treatment(args.baseline)
+    names = [require_treatment(name, label="candidate treatment") for name in args.candidates]
+    candidates = list(dict.fromkeys(names))
+    for name in candidates:
+        if names.count(name) > 1:
+            print(f"cv: warning: {name} is listed more than once; reading it once", file=sys.stderr)
+    noise = (
+        require_treatment(args.noise_floor, label="noise-floor treatment")
+        if args.noise_floor
+        else None
+    )
+    params = {"basis": args.basis} if args.basis == "pre_audit" else None
+
+    def compare(candidate: str) -> Mapping[str, object] | str:
+        path = (
+            f"{_TREATMENTS_PATH}/{urllib.parse.quote(baseline, safe='')}"
+            f"/comparisons/{urllib.parse.quote(candidate, safe='')}"
+        )
+        try:
+            payload = read(path, params)
+        except ControlRefusedError as exc:
+            return str(exc)
+        except NetworkError as exc:
+            return f"request_failed: {exc}"
+        warn(payload)
+        return payload
+
+    noise_result = compare(noise) if noise is not None else None
+    results = {name: compare(name) for name in candidates}
+    rows = frontier_rows(baseline, results, args.batch, noise=noise, noise_result=noise_result)
+    subset = [row["treatment"] for row in rows if row["cost_scope"] == "batch subset"]
+    if subset:
+        print(
+            f"cv: warning: {', '.join(subset)} grade a different batch set than the baseline, "
+            "so their cost covers only the shared batches and they are not ranked. "
+            "Compare one batch with --batch.",
+            file=sys.stderr,
+        )
+    if args.format == "json":
+        emit_json(rows)
+    else:
+        print(_text(rows, baseline, noise, args), end="")
+    if all(row["reason"] is not None for row in rows if row["role"] == "candidate"):
+        print("cv: every candidate failed", file=sys.stderr)
+        return int(ExitCode.REFUSAL)
+    return 0
+
+
+def frontier_rows(
+    baseline: str,
+    results: Mapping[str, Mapping[str, object] | str],
+    batch: str | None,
+    *,
+    noise: str | None = None,
+    noise_result: Mapping[str, object] | str | None = None,
+) -> list[Row]:
+    """Baseline, noise-floor and candidate rows, each built from one compare response."""
+
+    rows = [_baseline_row(baseline, [*results.values(), noise_result], batch)]
+    noise_row = None
+    if noise is not None and noise_result is not None:
+        noise_row = _compared_row(noise, "noise_floor", noise_result, batch)
+        rows.append(noise_row)
+    for name, result in results.items():
+        row = _compared_row(name, "candidate", result, batch)
+        if row["reason"] is None:
+            row["within_noise"] = _within_noise(row, noise_row)
+        rows.append(row)
+    scope = rows[0]["cost_scope"]
+    placed = [
+        (row, position)
+        for row in rows
+        if row["role"] != "noise_floor" and (position := _position(row, scope)) is not None
+    ]
+    for row in rows:
+        if row["role"] == "noise_floor":
+            continue
+        mine = _position(row, scope)
+        row["frontier"] = mine is not None and not any(
+            _dominates(theirs, mine) for other, theirs in placed if other is not row
+        )
+    return rows
+
+
+def _row(treatment: str, role: str) -> Row:
+    return {
+        "treatment": treatment,
+        "role": role,
+        "changed_dimensions": [],
+        "delta_tpr": None,
+        "delta_tnr": None,
+        "cost_usd": None,
+        "cost_basis": "unknown",
+        "extraction_usd": None,
+        "extraction_cost_basis": "unknown",
+        "grading_usd": None,
+        "grading_cost_basis": "unknown",
+        "price_basis_label": "price basis unconfirmed",
+        "cost_scope": "whole run",
+        "within_noise": None,
+        "frontier": None,
+        "reason": None,
+    }
+
+
+def _baseline_row(
+    baseline: str, results: Sequence[Mapping[str, object] | str | None], batch: str | None
+) -> Row:
+    """Zero deltas. The cost is the batch's when any compare reports it, else the whole run's."""
+
+    row = _row(baseline, "baseline")
+    zero = {"point": 0.0, "lower": None, "upper": None}
+    row["delta_tpr"] = zero
+    row["delta_tnr"] = dict(zero)
+    costs = [_cost(result, "baseline", batch) for result in results if isinstance(result, Mapping)]
+    chosen = next((cost for cost in costs if cost[2] == "batch"), costs[0] if costs else None)
+    if chosen is not None:
+        row["cost_usd"], row["cost_basis"], row["cost_scope"] = chosen
+    row["latency"] = next(
+        (
+            _latency(result, "baseline", batch)
+            for result in results
+            if isinstance(result, Mapping) and _latency(result, "baseline", batch) is not None
+        ),
+        None,
+    )
+    if row["latency"] is None:
+        row.pop("latency")
+    source = next((result for result in results if isinstance(result, Mapping)), None)
+    if source is not None:
+        _split(row, source, "baseline", batch)
+    return row
+
+
+def _compared_row(
+    treatment: str, role: str, result: Mapping[str, object] | str, batch: str | None
+) -> Row:
+    row = _row(treatment, role)
+    if isinstance(result, str):
+        row["reason"] = result
+        return row
+    row["changed_dimensions"] = [
+        str(dimension.get("dimension")) for dimension in _mappings(result.get("changed_dimensions"))
+    ]
+    row["cost_usd"], row["cost_basis"], row["cost_scope"] = _cost(result, "candidate", batch)
+    if result.get("latency") is not None:
+        row["latency"] = _latency(result, "candidate", batch)
+    _split(row, result, "candidate", batch)
+    source: object = result.get("pooled")
+    if batch is not None:
+        match = next(
+            (
+                item
+                for item in _mappings(result.get("batches"))
+                if item.get("batch_member_id") == batch
+            ),
+            None,
+        )
+        if match is None:
+            row["reason"] = f"batch {batch} is not in this comparison"
+            return row
+        if match.get("comparable") is not True:
+            row["reason"] = f"batch {batch} is not comparable: {match.get('reason')}"
+            return row
+        source = match.get("difference")
+    if isinstance(source, Mapping):
+        row["delta_tpr"] = _interval(source.get("tpr"))
+        row["delta_tnr"] = _interval(source.get("tnr"))
+    return row
+
+
+def _interval(metric: object) -> dict[str, object] | None:
+    micro = metric.get("micro") if isinstance(metric, Mapping) else None
+    if not isinstance(micro, Mapping):
+        return None
+    return {key: micro.get(key) for key in ("point", "lower", "upper")}
+
+
+def _cost(
+    payload: Mapping[str, object], side: str, batch: str | None
+) -> tuple[float | None, str, str]:
+    """Amount, basis and scope: the batch's call cost when present, else the whole run's.
+
+    Whole-arm ranking uses the total per trace regardless of compare's legacy
+    subset cost delta. Batch ranking uses that batch's production split. Exact
+    and upper-bound costs are placed. Lower bounds are displayed but never ranked.
+    """
+
+    if batch is not None:
+        call = next(
+            (
+                row
+                for row in _mappings(payload.get("batch_production_costs"))
+                if row.get("batch_member_id") == batch
+            ),
+            None,
+        )
+        split = call.get("cost") if call is not None else None
+        total = split.get("total") if isinstance(split, Mapping) else None
+        cost = total.get(side) if isinstance(total, Mapping) else None
+        if isinstance(cost, Mapping):
+            return (*_placed_amount(cost.get("amount"), cost.get("cost_basis")), "batch")
+    amount = _arm_production_equivalent(payload, side)
+    arm = payload.get(f"{side}_arm_cost")
+    if isinstance(arm, Mapping) and isinstance(arm.get("total_per_trace"), Mapping):
+        amount = arm["total_per_trace"]
+    return (*_placed_amount(amount.get("amount"), amount.get("cost_basis")), "whole run")
+
+
+def _split(row: Row, payload: Mapping[str, object], side: str, batch: str | None) -> None:
+    if batch is not None and row["cost_scope"] == "batch":
+        matched = next(
+            (
+                entry
+                for entry in _mappings(payload.get("batch_production_costs"))
+                if entry.get("batch_member_id") == batch
+            ),
+            {},
+        )
+        cost = matched.get("cost")
+        if isinstance(cost, Mapping):
+            for name in ("extraction", "grading"):
+                delta = cost.get(name)
+                part = delta.get(side) if isinstance(delta, Mapping) else None
+                if isinstance(part, Mapping):
+                    row[f"{name}_usd"], row[f"{name}_cost_basis"] = _placed_amount(
+                        part.get("amount"), part.get("cost_basis")
+                    )
+        return
+    arm = payload.get(f"{side}_arm_cost")
+    if not isinstance(arm, Mapping):
+        return
+    distribution = arm.get("distribution")
+    if isinstance(distribution, Mapping):
+        total = distribution.get("total")
+        p90 = total.get("p90") if isinstance(total, Mapping) else None
+        row["p90_total_usd"] = p90.get("amount") if isinstance(p90, Mapping) else None
+        row["p90_total_precision"] = p90.get("precision") if isinstance(p90, Mapping) else "unknown"
+    for name, key in (
+        ("extraction", "extraction_per_trace"),
+        ("grading", "grading_per_production_grade"),
+    ):
+        part = arm.get(key)
+        if isinstance(part, Mapping):
+            row[f"{name}_usd"], row[f"{name}_cost_basis"] = _placed_amount(
+                part.get("amount"), part.get("cost_basis")
+            )
+            row["price_basis_label"] = part.get("price_basis_label", "price basis unconfirmed")
+
+
+def _arm_production_equivalent(payload: Mapping[str, object], side: str) -> Mapping[str, object]:
+    arm = payload.get(f"{side}_arm_cost")
+    amount = arm.get("production_equivalent_per_trace") if isinstance(arm, Mapping) else None
+    return amount if isinstance(amount, Mapping) else {}
+
+
+def _placed_amount(value: object, basis: object) -> tuple[float | None, str]:
+    if basis in (*_PLACED_COST_BASES, "lower bound") and _is_number(value):
+        return float(value), str(basis)
+    return None, "unknown"
+
+
+def _within_noise(row: Row, noise_row: Row | None) -> bool | str:
+    """Both delta points inside the noise floor's paired intervals."""
+
+    if noise_row is None or noise_row["reason"] is not None:
+        return NOT_MEASURED
+    checks: list[bool] = []
+    for key in ("delta_tpr", "delta_tnr"):
+        delta = row[key] or {}
+        bounds = noise_row[key] or {}
+        point, lower, upper = delta.get("point"), bounds.get("lower"), bounds.get("upper")
+        if not (_is_number(point) and _is_number(lower) and _is_number(upper)):
+            return NOT_MEASURED
+        checks.append(lower <= point <= upper)
+    return all(checks)
+
+
+def _position(row: Row, scope: str) -> tuple[float, float, float] | None:
+    """Negated cost, delta TPR and delta TNR, so larger is better on all three.
+
+    A row with a reason, an unknown cost, a cost of another scope than the
+    baseline's, or an unknown delta has no position.
+    """
+
+    cost = row["cost_usd"]
+    tpr, tnr = _point(row, "delta_tpr"), _point(row, "delta_tnr")
+    if (
+        row["reason"] is not None
+        or row["cost_scope"] != scope
+        or row["cost_basis"] not in _PLACED_COST_BASES
+    ):
+        return None
+    if cost is None or tpr is None or tnr is None:
+        return None
+    return (-cost, tpr, tnr)
+
+
+def _dominates(theirs: tuple[float, ...], mine: tuple[float, ...]) -> bool:
+    """At least as cheap and as good on both deltas, and strictly better on one."""
+
+    return theirs != mine and all(a >= b for a, b in zip(theirs, mine, strict=True))
+
+
+def _point(row: Row, key: str) -> float | None:
+    delta = row[key]
+    point = delta.get("point") if isinstance(delta, Mapping) else None
+    return float(point) if _is_number(point) else None
+
+
+def _text(rows: Sequence[Row], baseline: str, noise: str | None, args: argparse.Namespace) -> str:
+    heading = f"frontier baseline={baseline} noise_floor={noise or NOT_MEASURED} basis={args.basis}"
+    if args.batch:
+        heading += f" batch={args.batch}"
+    lines = [heading, "note=This ranks treatments. The swap bar decides replacement."]
+    has_latency = any(row.get("latency") is not None for row in rows)
+    for row in rows:
+        bits = [f"treatment={row['treatment']}", f"role={row['role']}"]
+        if row["reason"] is not None:
+            lines.append(" ".join([*bits, f"reason={row['reason']}"]))
+            continue
+        bits.append("changed=" + (",".join(row["changed_dimensions"]) or "none"))
+        bits.append(f"delta_tpr={_delta_text(row['delta_tpr'])}")
+        bits.append(f"delta_tnr={_delta_text(row['delta_tnr'])}")
+        cost = row["cost_usd"]
+        bits.append(
+            "cost unknown" if cost is None else f"cost_usd={_number(cost)} {row['cost_basis']}"
+        )
+        if "p90_total_usd" in row:
+            p90 = row["p90_total_usd"]
+            bits.append(
+                f"p90_total_usd={_number(p90) if _is_number(p90) else 'unknown'} "
+                f"precision={row['p90_total_precision']}"
+            )
+        bits.append(f"cost_scope={row['cost_scope']}")
+        if has_latency:
+            latency = row.get("latency")
+            if latency is not None:
+                bits.append("latency_basis=bucket_upper_bound latency_scope=batch_executions")
+            for name in ("p50", "p90"):
+                value = latency.get(f"{name}_seconds") if isinstance(latency, Mapping) else None
+                shown = (
+                    _number(value)
+                    if _is_number(value)
+                    else (
+                        ">1800 s"
+                        if isinstance(latency, Mapping) and latency.get("overflow")
+                        else "unknown"
+                    )
+                )
+                bits.append(f"latency_{name}_seconds={shown}")
+        for name in ("extraction", "grading"):
+            amount = row[f"{name}_usd"]
+            bits.append(
+                f"{name}_usd={_number(amount) if amount is not None else 'unknown'} "
+                f"{row[f'{name}_cost_basis']}"
+            )
+        bits.append(row["price_basis_label"])
+        if row["role"] == "candidate":
+            bits.append(f"within_noise={_flag(row['within_noise'])}")
+        if row["role"] != "noise_floor":
+            bits.append(f"frontier={_flag(row['frontier'])}")
+        lines.append(" ".join(bits))
+    return "\n".join(lines) + "\n"
+
+
+def _delta_text(delta: object) -> str:
+    if not isinstance(delta, Mapping) or not _is_number(delta.get("point")):
+        return "unknown"
+    text = _number(delta["point"])
+    lower, upper = delta.get("lower"), delta.get("upper")
+    if _is_number(lower) and _is_number(upper):
+        text += f" [{_number(lower)}, {_number(upper)}]"
+    return text
+
+
+def _flag(value: object) -> str:
+    return str(value).lower() if isinstance(value, bool) else str(value)
+
+
+def _number(value: float) -> str:
+    return f"{value:.6g}"
+
+
+def _is_number(value: object) -> TypeGuard[int | float]:
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
+def _mappings(value: object) -> list[Mapping[str, object]]:
+    return [item for item in value if isinstance(item, Mapping)] if isinstance(value, list) else []
+
+
+def _latency(payload: Mapping[str, object], side: str, batch: str | None) -> object:
+    comparison = payload.get("latency")
+    if batch is not None:
+        comparison = next(
+            (
+                row.get("latency")
+                for row in _mappings(payload.get("batch_call_costs"))
+                if row.get("batch_member_id") == batch
+            ),
+            None,
+        )
+    return comparison.get(side) if isinstance(comparison, Mapping) else None