downshift 0.1.0.dev0__py3-none-any.whl

downshift/decide.py

@@ -0,0 +1,220 @@
+"""Downgrade decisions: the cheapest model that keeps enough of the baseline's quality.
+
+Decisions use pass rate (share of cases fully right), never mean score. A model can get
+most JSON fields right and still fail most cases.
+
+A candidate qualifies when all of these hold:
+- its results are complete (every case scored, no error rows),
+- pass rate / baseline pass rate >= threshold,
+- pass rate >= min_pass_rate (absolute floor, 0 = off),
+- it costs less per call than the baseline (priced with its own measured tokens).
+The cheapest qualifying candidate wins; ties go to the higher pass rate, then list order.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from downshift.config import ModelPrice
+from downshift.evals import EvalSet
+from downshift.runner import ResultRow, RunSummary, load_results, results_path, summarize_rows
+
+KEEP = "keep"
+DOWNGRADE = "downgrade"
+EPSILON = 1e-9  # 18/22 vs 20/22 must count as exactly 0.90
+
+
+def call_cost(price: ModelPrice, prompt_tokens: float, completion_tokens: float) -> float:
+    """USD for one call with these (average) token counts."""
+    total = prompt_tokens * price.input_per_mtok + completion_tokens * price.output_per_mtok
+    return total / 1_000_000
+
+
+@dataclass(frozen=True)
+class ModelStats:
+    """Results of one model on one call site."""
+
+    summary: RunSummary
+    avg_judge_score: float | None = None
+
+    @property
+    def model(self) -> str:
+        return self.summary.model
+
+    @property
+    def pass_rate(self) -> float | None:
+        return self.summary.pass_rate
+
+    @property
+    def complete(self) -> bool:
+        s = self.summary
+        return s.cases > 0 and s.errors == 0 and s.scored == s.cases
+
+    def cost_per_call(self, price: ModelPrice) -> float | None:
+        s = self.summary
+        if s.avg_prompt_tokens is None or s.avg_completion_tokens is None:
+            return None
+        return call_cost(price, s.avg_prompt_tokens, s.avg_completion_tokens)
+
+
+def stats_for(
+    site_id: str, model: str, case_ids: Sequence[str], rows: Mapping[str, ResultRow]
+) -> ModelStats:
+    """Stats for these case ids. Avg judge score counts only scored rows that have one."""
+    summary = summarize_rows(site_id, model, case_ids, rows)
+    judged = [
+        float(rows[c].judge_score)  # type: ignore[arg-type]
+        for c in case_ids
+        if c in rows and rows[c].ok and rows[c].judge_score is not None
+    ]
+    avg = sum(judged) / len(judged) if judged else None
+    return ModelStats(summary=summary, avg_judge_score=avg)
+
+
+def load_site_stats(
+    site_id: str, eval_set: EvalSet, models: Iterable[str], results_dir: Path
+) -> dict[str, ModelStats]:
+    """Stats per model for one site, from results files (last row per case wins)."""
+    case_ids = [case.id for case in eval_set.cases]
+    out: dict[str, ModelStats] = {}
+    for model in models:
+        rows = load_results(results_path(results_dir, site_id, model))
+        out[model] = stats_for(site_id, model, case_ids, rows)
+    return out
+
+
+@dataclass(frozen=True)
+class CandidateCheck:
+    """Why one candidate did or did not qualify."""
+
+    model: str
+    pass_rate: float | None
+    ratio: float | None
+    cost_per_call: float | None
+    passed: bool
+    reason: str
+
+
+@dataclass(frozen=True)
+class Decision:
+    site_id: str
+    action: str  # KEEP or DOWNGRADE
+    model: str  # the model to use after the decision
+    baseline: str
+    reason: str
+    baseline_pass_rate: float | None
+    baseline_below_floor: bool
+    checks: tuple[CandidateCheck, ...]
+    stats: Mapping[str, ModelStats]
+
+    @property
+    def downgraded(self) -> bool:
+        return self.action == DOWNGRADE
+
+
+def _price(prices: Mapping[str, ModelPrice], model: str) -> ModelPrice:
+    try:
+        return prices[model]
+    except KeyError:
+        raise ValueError(f"no pricing for model {model!r}") from None
+
+
+def _incomplete(s: RunSummary) -> str:
+    return f"incomplete results ({s.errors} errors, {s.scored}/{s.cases} scored)"
+
+
+def _check(
+    model: str,
+    s: ModelStats | None,
+    *,
+    base_rate: float,
+    base_cost: float | None,
+    prices: Mapping[str, ModelPrice],
+    threshold: float,
+    min_pass_rate: float,
+) -> CandidateCheck:
+    if s is None or s.summary.scored == 0:
+        return CandidateCheck(model, None, None, None, False, "no results")
+    rate = s.pass_rate or 0.0
+    ratio = rate / base_rate
+    cost = s.cost_per_call(_price(prices, model))
+
+    def fail(reason: str) -> CandidateCheck:
+        return CandidateCheck(model, rate, ratio, cost, False, reason)
+
+    if not s.complete:
+        return fail(_incomplete(s.summary))
+    if ratio < threshold - EPSILON:
+        return fail(f"keeps {ratio:.0%} of baseline quality, needs {threshold:.0%}")
+    if rate < min_pass_rate - EPSILON:
+        return fail(f"pass rate {rate:.0%} is below the floor {min_pass_rate:.0%}")
+    if cost is None or base_cost is None or cost >= base_cost:
+        return fail("not cheaper than the baseline")
+    return CandidateCheck(
+        model, rate, ratio, cost, True, f"keeps {ratio:.0%} of baseline quality and costs less"
+    )
+
+
+def decide_site(
+    site_id: str,
+    stats: Mapping[str, ModelStats],
+    *,
+    baseline: str,
+    candidates: Sequence[str],
+    prices: Mapping[str, ModelPrice],
+    threshold: float,
+    min_pass_rate: float = 0.0,
+) -> Decision:
+    """Keep the baseline or downgrade to the cheapest qualifying candidate."""
+    if not 0.0 < threshold <= 1.0:
+        raise ValueError(f"threshold must be in (0, 1], got {threshold}")
+    if not 0.0 <= min_pass_rate <= 1.0:
+        raise ValueError(f"min_pass_rate must be in [0, 1], got {min_pass_rate}")
+
+    def keep(
+        reason: str,
+        base_rate: float | None = None,
+        below: bool = False,
+        checks: tuple[CandidateCheck, ...] = (),
+    ) -> Decision:
+        return Decision(site_id, KEEP, baseline, baseline, reason, base_rate, below, checks, stats)
+
+    base = stats.get(baseline)
+    if base is None or base.summary.scored == 0:
+        return keep("no baseline results")
+    if not base.complete:
+        return keep(f"baseline has {_incomplete(base.summary)}", base.pass_rate)
+    base_rate = base.pass_rate or 0.0
+    below = base_rate < min_pass_rate - EPSILON
+    if base_rate == 0.0:
+        return keep("baseline passes no cases; nothing to compare against", 0.0, below)
+
+    base_cost = base.cost_per_call(_price(prices, baseline))
+    checks = tuple(
+        _check(
+            model,
+            stats.get(model),
+            base_rate=base_rate,
+            base_cost=base_cost,
+            prices=prices,
+            threshold=threshold,
+            min_pass_rate=min_pass_rate,
+        )
+        for model in candidates
+        if model != baseline
+    )
+    order = {c.model: i for i, c in enumerate(checks)}
+    passing = [c for c in checks if c.passed]
+    if not passing:
+        reason = "no candidate models" if not checks else "no cheaper model passed the checks"
+        return keep(reason, base_rate, below, checks)
+
+    best = min(
+        passing,
+        key=lambda c: (c.cost_per_call or 0.0, -(c.pass_rate or 0.0), order[c.model]),
+    )
+    return Decision(
+        site_id, DOWNGRADE, best.model, baseline, best.reason, base_rate, below, checks, stats
+    )

downshift/evalgen.py

@@ -0,0 +1,220 @@
+"""Generate eval sets with any configured model (the no-Bob path).
+
+`generate_eval_set` asks a model for test cases for one call site, then keeps
+only the cases that pass the same checks as `downshift check-evals`. Cases are
+renumbered, so the model never has to get ids right. The CLI command
+`evalgen` is a thin wrapper that writes one `<slug>.jsonl` per call site.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from downshift.evals import (
+    EvalCase,
+    EvalSet,
+    allowed_values,
+    expression_placeholders,
+    graded_fields,
+    site_placeholders,
+    validate_eval_set,
+)
+from downshift.llm import LLMClient
+from downshift.schema import CallSite
+
+_FENCE = re.compile(r"^```(?:json)?\s*|\s*```$")
+
+SYSTEM_PROMPT = (
+    "You write test cases for one LLM feature in a software product. "
+    "Every expected answer must be certainly correct. Reply with JSON only."
+)
+
+
+class EvalGenSkip(Exception):
+    """Raised when a call site cannot get generated evals; the message says why."""
+
+
+@dataclass
+class GenerationResult:
+    site_id: str
+    grading: str
+    cases: list[EvalCase] = field(default_factory=list)
+    dropped: list[str] = field(default_factory=list)
+    attempts: int = 0
+
+
+def grading_for(site: CallSite) -> str:
+    """The audit's grading, or a safe default for an unaudited call site."""
+    if site.grading:
+        return site.grading
+    return "json_fields" if site.output_format == "json" else "judge"
+
+
+def generate_eval_set(
+    client: LLMClient,
+    model: str,
+    site: CallSite,
+    *,
+    count: int = 20,
+    shared: dict[str, str] | None = None,
+    max_attempts: int = 3,
+    temperature: float = 0.7,
+) -> GenerationResult:
+    """Ask `model` for cases until `count` valid ones are kept or attempts run out."""
+    _check_site(site)
+    shared = {k: v for k, v in (shared or {}).items() if k in site_placeholders(site)}
+    grading = grading_for(site)
+    result = GenerationResult(site_id=site.id, grading=grading)
+    seen_inputs: set[str] = set()
+
+    while len(result.cases) < count and result.attempts < max_attempts:
+        result.attempts += 1
+        wanted = count - len(result.cases)
+        messages = build_messages(site, grading, wanted, shared)
+        completion = client.complete(model, messages, temperature=temperature, json_mode=True)
+        try:
+            raw_cases = parse_cases(completion.text)
+        except ValueError as exc:
+            result.dropped.append(f"attempt {result.attempts}: {exc}")
+            continue
+        for raw in raw_cases:
+            if len(result.cases) >= count:
+                break
+            case, problem = _to_case(raw, site, grading, shared, len(result.cases) + 1)
+            if case is None:
+                result.dropped.append(problem)
+                continue
+            key = json.dumps(case.inputs, sort_keys=True, ensure_ascii=False)
+            if key in seen_inputs:
+                result.dropped.append(f"duplicate inputs: {case.notes}")
+                continue
+            seen_inputs.add(key)
+            result.cases.append(case)
+    return result
+
+
+def build_messages(
+    site: CallSite, grading: str, count: int, shared: dict[str, str]
+) -> list[dict[str, str]]:
+    inputs = [name for name in site_placeholders(site) if name not in shared]
+    prompt_lines = [f"[{m.role}]\n{m.content}" for m in site.messages or []]
+    parts = [
+        f"Feature: {site.purpose or site.function}",
+        f"Output contract: {site.output_contract or 'not documented'}",
+        "Prompt template sent to the model ({name} is filled per case):",
+        "\n\n".join(prompt_lines),
+    ]
+    if shared:
+        parts.append("Fixed inputs, the same for every case (do not repeat them in cases):")
+        parts.extend(f"--- {k} ---\n{v}" for k, v in shared.items())
+    parts.append(f"Write {count} diverse, realistic test cases.")
+    parts.append(
+        f"Each case has keys: inputs (an object with exactly these keys: "
+        f"{', '.join(inputs)}), expected, notes (what the case tests)."
+    )
+    parts.append(_expected_rule(site, grading))
+    parts.append(
+        "Include edge cases: very short text, other languages, typos, "
+        "missing details. Skip any case whose answer could be argued."
+    )
+    parts.append('Reply with: {"cases": [ ... ]}')
+    return [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user", "content": "\n\n".join(parts)},
+    ]
+
+
+def parse_cases(text: str) -> list[Any]:
+    """Pull the case list out of a model reply. Raises ValueError if there is none."""
+    cleaned = _FENCE.sub("", text.strip())
+    try:
+        data = json.loads(cleaned)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"reply is not valid JSON ({exc.msg})") from exc
+    if isinstance(data, dict):
+        data = data.get("cases")
+    if not isinstance(data, list):
+        raise ValueError('reply has no "cases" list')
+    return data
+
+
+def write_eval_set(
+    path: Path, cases: list[EvalCase], shared_files: dict[str, Path] | None = None
+) -> None:
+    """Write cases as JSONL, with a `shared` first line pointing at files if given."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lines = []
+    if shared_files:
+        header = {
+            key: {"file": Path(os.path.relpath(target, path.parent)).as_posix()}
+            for key, target in shared_files.items()
+        }
+        lines.append(json.dumps({"shared": header}, ensure_ascii=False))
+    for case in cases:
+        record = {
+            "id": case.id,
+            "inputs": case.inputs,
+            "expected": case.expected,
+            "grading": case.grading,
+            "notes": case.notes,
+        }
+        lines.append(json.dumps(record, ensure_ascii=False))
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+# --- helpers ------------------------------------------------------------------
+
+
+def _check_site(site: CallSite) -> None:
+    if not site.prompt_resolved:
+        raise EvalGenSkip("prompt is not resolved; run the Bob auditor first")
+    expressions = expression_placeholders(site)
+    if expressions:
+        names = ", ".join("{" + e + "}" for e in expressions)
+        raise EvalGenSkip(f"prompt has expression placeholders {names}; run the Bob auditor first")
+
+
+def _expected_rule(site: CallSite, grading: str) -> str:
+    if grading == "exact":
+        labels = allowed_values(site)
+        if labels:
+            return f"expected: exactly one of {', '.join(labels)} (a string)."
+        return "expected: the exact short answer as a string."
+    if grading == "json_fields":
+        fields = graded_fields(site)
+        if fields:
+            return f"expected: an object with exactly these keys: {', '.join(fields)}."
+        return "expected: the JSON object the feature should return."
+    return (
+        "expected: a rubric string listing what a correct reply must and must not do, "
+        "point by point."
+    )
+
+
+def _to_case(
+    raw: Any, site: CallSite, grading: str, shared: dict[str, str], number: int
+) -> tuple[EvalCase | None, str]:
+    if not isinstance(raw, dict):
+        return None, f"not an object: {str(raw)[:60]}"
+    notes = raw.get("notes")
+    notes = notes if isinstance(notes, str) and notes.strip() else "generated"
+    inputs = raw.get("inputs")
+    if not isinstance(inputs, dict):
+        return None, f"inputs is not an object: {notes}"
+    case = EvalCase(
+        id=f"{site.function}-{number:02d}",
+        inputs=inputs,
+        expected=raw.get("expected"),
+        grading=grading,
+        notes=notes,
+    )
+    check = validate_eval_set(EvalSet(path=Path("-"), cases=[case], shared=shared), site)
+    if check.errors:
+        reason = check.errors[0].split(": ", 1)[-1]
+        return None, f"{reason}: {notes}"
+    return case, ""