promptcheckup 1.0.0__py3-none-any.whl

promptcheckup/__init__.py

@@ -0,0 +1,7 @@
+"""Prove a prompt change made your AI agent better, with paired replay."""
+
+from promptcheckup.data import record
+from promptcheckup.stats import Verdict, decide
+
+__all__ = ["Verdict", "decide", "record"]
+__version__ = "1.0.0"

promptcheckup/checkup/__init__.py

@@ -0,0 +1,62 @@
+"""Shared evaluation and human-readable reports."""
+
+from promptcheckup.checkup.checkup import CheckupResult, checkup, propose_only
+from promptcheckup.checkup.evaluate import (
+    Evaluation,
+    Scorer,
+    Scores,
+    evaluate,
+    evaluate_replay,
+    judge_mode,
+    planned_calls,
+    redacted_inputs,
+    score_mode,
+)
+from promptcheckup.checkup.evolve import (
+    EvolveResult,
+    Round,
+    evolution_cases,
+    evolve,
+    round_confidence,
+    split_cases,
+)
+from promptcheckup.checkup.ledger import false_better_expected, record_trial, trials
+from promptcheckup.checkup.report import checkup_report, evolve_report, gate_report, proposal_report
+from promptcheckup.checkup.snapshot import (
+    Snapshot,
+    load_latest_snapshot,
+    load_snapshot,
+    save_snapshot,
+)
+
+__all__ = [
+    "EvolveResult",
+    "Round",
+    "evolution_cases",
+    "evolve",
+    "split_cases",
+    "round_confidence",
+    "evolve_report",
+    "record_trial",
+    "trials",
+    "false_better_expected",
+    "CheckupResult",
+    "Snapshot",
+    "checkup",
+    "checkup_report",
+    "evaluate_replay",
+    "load_snapshot",
+    "load_latest_snapshot",
+    "proposal_report",
+    "propose_only",
+    "save_snapshot",
+    "Evaluation",
+    "Scorer",
+    "Scores",
+    "evaluate",
+    "gate_report",
+    "judge_mode",
+    "planned_calls",
+    "redacted_inputs",
+    "score_mode",
+]

promptcheckup/checkup/checkup.py

@@ -0,0 +1,336 @@
+"""Weekly check-up and proposal pipelines: named steps over one frozen state and budget.
+
+Each step takes the fixed plan and the state so far and returns a new state. A step
+that cannot afford its calls stops the pipeline; the result then says the budget ran
+out, keeps the evidence gathered so far, and never keeps a proposal.
+"""
+
+from collections.abc import Callable
+from dataclasses import dataclass, field, replace
+from functools import reduce
+from pathlib import Path
+from typing import Any
+
+from promptcheckup.checkup.evaluate import (
+    Evaluation,
+    Scorer,
+    evaluate,
+    evaluate_replay,
+    judge_mode,
+    redacted_inputs,
+    score_mode,
+)
+from promptcheckup.checkup.failures import current_failures
+from promptcheckup.checkup.ledger import record_trial
+from promptcheckup.checkup.snapshot import load_latest_snapshot, load_snapshot, save_snapshot
+from promptcheckup.data import Case, append_cases
+from promptcheckup.errors import errors_as, fallback
+from promptcheckup.llm.patterns import Pattern, failure_patterns
+from promptcheckup.llm.propose import Proposal, ProposalRejected, propose
+from promptcheckup.llm.starter import starter_cases
+from promptcheckup.runner import Agent, AgentError, Cache, Replay, Run, prompt_hash, run_prompts
+from promptcheckup.stats import Verdict
+
+
+@dataclass(frozen=True)
+class CheckupResult:
+    starter_created: int
+    drift: Evaluation | None
+    agent_version_changed: bool
+    patterns: list[Pattern]
+    proposal: Proposal | None
+    proposal_verdict: Verdict | None
+    proposal_rejection: str | None
+    calls: int
+    synthetic_share: float = 0.0
+    budget_exhausted: bool = False
+    errors: list[Run] = field(default_factory=list)
+    prompts_hash: str = ""
+    model: str = ""
+    drift_checked: bool = True
+    snapshot_saved: bool = False
+    gated_proposal: Proposal | None = None  # audit evidence, never authorization to write files
+    drift_baseline: str = "same prompt"
+    previous_prompts_hash: str = ""
+
+
+@dataclass(frozen=True)
+class _Plan:
+    """Everything fixed for one run. `score_fn` selects score mode; otherwise judge mode."""
+
+    agent: Agent
+    prompts: dict[str, str]
+    llm: Callable[[str], str]
+    directory: Path
+    score_fn: Callable[[Any, Any], float] | None = None
+    criteria: str = ""
+    scorer: Scorer | None = None
+    samples: int = 2
+    timeout_s: float = 60
+    max_calls: int = 400
+    min_pairs: int = 20
+    cache: Cache | None = None
+    agent_version: str = ""
+    description: str = ""
+    example_input: dict = field(default_factory=dict)
+    redact: Callable[[Any], Any] | None = None
+    gate_cases: list[Case] | None = None
+    confidence: float = 0.95
+    trial_command: str | None = None  # CLI opts into durable recording immediately after gating
+
+
+@dataclass(frozen=True)
+class _State:
+    result: CheckupResult
+    cases: list[Case]
+    current: Replay = field(default_factory=lambda: Replay([], 0, False))
+    failures: list[tuple] = field(default_factory=list)
+    stopped: bool = False
+
+
+def checkup(agent, cases, prompts, llm, **settings) -> CheckupResult:
+    """Starter cases, current runs, drift, failures, patterns, and a gated proposal.
+
+    Settings are the fields of `_Plan`: directory (project root), score_fn or criteria,
+    samples, timeout_s, max_calls, min_pairs, cache, agent_version, description,
+    example_input, redact, and optionally a scorer following evaluate's protocol.
+    gate_cases and confidence override only the proposal gate. trial_command opts
+    into recording each gate immediately, including rejected and budget-stopped gates.
+    """
+    plan = _plan(agent, prompts, llm, settings)
+    return _run(plan, _start(plan, cases), CHECKUP_STEPS)
+
+
+def propose_only(agent, cases, prompts, llm, **settings) -> CheckupResult:
+    """Current runs, failures, patterns and the gate, without starter cases or drift."""
+    if not cases:
+        raise ValueError("propose requires existing cases")
+    plan = _plan(agent, prompts, llm, settings)
+    state = _start(plan, cases)
+    return _run(plan, replace(state, result=replace(state.result, drift_checked=False)), PROPOSE)
+
+
+def _plan(agent, prompts, llm, settings) -> _Plan:
+    plan = _Plan(agent, prompts, llm, **settings)
+    # Outputs are redacted as they leave the agent: before the cache, snapshots and the LLM.
+    return replace(
+        plan,
+        directory=Path(plan.directory),
+        agent=_redacting(agent, plan.redact),
+        scorer=_scorer(plan),
+    )
+
+
+def _start(plan, cases) -> _State:
+    model = getattr(plan.llm, "model", "")
+    result = CheckupResult(
+        0, None, False, [], None, None, None, 0, prompts_hash=prompt_hash(plan.prompts), model=model
+    )
+    return _State(result, list(cases))
+
+
+def _run(plan, state, steps) -> CheckupResult:
+    final = reduce(lambda done, step: done if done.stopped else step(plan, done), steps, state)
+    if final.stopped:
+        return replace(final.result, budget_exhausted=True, proposal=None)
+    return final.result
+
+
+# Steps ---------------------------------------------------------------------------------
+
+
+def _starter(plan, state) -> _State:
+    if state.cases:
+        return state
+    if _remaining(plan, state) < 1:
+        return replace(state, stopped=True)
+    example = _redacted(plan.redact, plan.example_input)
+    generated = starter_cases(plan.llm, plan.prompts, plan.description, example)
+    cases = [replace(case, input=_redacted(plan.redact, case.input)) for case in generated]
+    append_cases(plan.directory / "cases" / "starter.jsonl", cases)
+    return _spend(replace(state, cases=cases), 1, starter_created=len(cases))
+
+
+def _current(plan, state) -> _State:
+    replayed = run_prompts(
+        plan.agent,
+        state.cases,
+        plan.prompts,
+        samples=plan.samples,
+        timeout_s=plan.timeout_s,
+        max_calls=_remaining(plan, state),
+        cache=plan.cache,
+    )
+    share = sum(case.source == "synthetic" for case in state.cases) / len(state.cases)
+    errors = [run for run in replayed.runs if run.error is not None]
+    state = _spend(state, replayed.calls, synthetic_share=share, errors=errors)
+    return replace(state, current=replayed, stopped=replayed.budget_exhausted)
+
+
+def _drift(plan, state) -> _State:
+    state = _compare_snapshot(plan, state)
+    if state.stopped:
+        return state  # an incomplete comparison keeps the previous snapshot
+    save_snapshot(plan.directory, state.result.prompts_hash, state.current.runs, plan.agent_version)
+    return _spend(state, 0, snapshot_saved=True)
+
+
+def _compare_snapshot(plan, state) -> _State:
+    previous = load_snapshot(plan.directory, state.result.prompts_hash)
+    if previous is not None:
+        return _compare_with(plan, state, previous)
+    latest = load_latest_snapshot(plan.directory, excluding=state.result.prompts_hash)
+    if latest is None:
+        return state
+    previous_hash, previous = latest
+    state = _spend(state, 0, drift_baseline="previous prompt", previous_prompts_hash=previous_hash)
+    return _compare_with(plan, state, previous)
+
+
+def _compare_with(plan, state, previous) -> _State:
+    ids = {case.id for case in state.cases}
+    old = [run for run in previous.runs if run.case_id in ids]
+    new = [replace(run, prompt_set="B") for run in state.current.runs]
+    drift = evaluate_replay(
+        state.cases,
+        Replay(old + new, 0, False),
+        plan.scorer,
+        budget=_remaining(plan, state),
+        min_pairs=plan.min_pairs,
+    )
+    changed = previous.agent_version != plan.agent_version
+    state = _spend(state, drift.scores.judge_calls, drift=drift, agent_version_changed=changed)
+    return replace(state, stopped=drift.budget_exhausted)
+
+
+def _failures(plan, state) -> _State:
+    judged = plan.score_fn is None
+    cost = sum(run.sample == 0 and run.error is None for run in state.current.runs) if judged else 0
+    if _remaining(plan, state) < cost:
+        return replace(state, stopped=True)
+    failures = current_failures(
+        _safe_cases(plan, state.cases) if judged else state.cases,
+        state.current,
+        score_fn=plan.score_fn,
+        llm=plan.llm,
+        criteria=plan.criteria,
+    )
+    safe = _safe_failures(plan, state.cases, failures)
+    return replace(_spend(state, cost), failures=safe)
+
+
+def _patterns(plan, state) -> _State:
+    cost = 1 if state.failures else 0
+    if _remaining(plan, state) < cost:
+        return replace(state, stopped=True)
+    return _spend(state, cost, patterns=failure_patterns(plan.llm, state.failures))
+
+
+def _proposal(plan, state) -> _State:
+    if not state.failures:
+        return state
+    if _remaining(plan, state) < 1:
+        return replace(state, stopped=True)
+    spent = _spend(state, 1)
+    # A rejected proposal is a finding for the report, not a failure of the run.
+    return fallback(
+        lambda: _gate(
+            plan, spent, propose(plan.llm, plan.prompts, spent.result.patterns, spent.failures)
+        ),
+        lambda error: _spend(spent, 0, proposal_rejection=str(error)),
+        ProposalRejected,
+    )
+
+
+def _gate(plan, state, proposal) -> _State:
+    # Replays current against proposed back-to-back; cached current outputs are free.
+    cases = state.cases if plan.gate_cases is None else plan.gate_cases
+    result = evaluate(
+        plan.agent,
+        cases,
+        plan.prompts,
+        proposal.prompts,
+        plan.scorer,
+        samples=plan.samples,
+        timeout_s=plan.timeout_s,
+        max_calls=_remaining(plan, state),
+        min_pairs=plan.min_pairs,
+        cache=plan.cache,
+        confidence=plan.confidence,
+    )
+    _record_trial(plan, cases, proposal, result)
+    gate_errors = [
+        run for run in result.errors if plan.gate_cases is not None or run.prompt_set == "B"
+    ]
+    state = _spend(
+        state,
+        result.replay.calls + result.scores.judge_calls,
+        proposal_verdict=result.verdict,
+        gated_proposal=proposal,
+        proposal=proposal if result.verdict.decision == "better" else None,
+        errors=[*state.result.errors, *gate_errors],
+    )
+    return replace(state, stopped=result.budget_exhausted)
+
+
+def _record_trial(plan, cases, proposal, result):
+    if plan.trial_command is not None:
+        record_trial(
+            plan.directory,
+            case_ids=[case.id for case in cases],
+            prompt_a=prompt_hash(plan.prompts),
+            prompt_b=prompt_hash(proposal.prompts),
+            decision=result.verdict.decision,
+            confidence=plan.confidence,
+            command=plan.trial_command,
+        )
+
+
+CHECKUP_STEPS = (_starter, _current, _drift, _failures, _patterns, _proposal)
+PROPOSE = (_current, _failures, _patterns, _proposal)
+
+
+# Helpers -------------------------------------------------------------------------------
+
+
+def _remaining(plan, state) -> int:
+    return plan.max_calls - state.result.calls
+
+
+def _spend(state, calls, **fields) -> _State:
+    result = replace(state.result, calls=state.result.calls + calls, **fields)
+    return replace(state, result=result)
+
+
+def _scorer(plan) -> Scorer:
+    if plan.scorer is not None:
+        return plan.scorer
+    if plan.score_fn is not None:
+        return score_mode(plan.score_fn)
+    return redacted_inputs(judge_mode(plan.llm, plan.criteria, cache=plan.cache), plan.redact)
+
+
+def _redacting(agent, redact) -> Agent:
+    if redact is None:
+        return agent
+    return lambda case, prompts, timeout_s: _redacted(redact, agent(case, prompts, timeout_s))
+
+
+def _redacted(redact, value):
+    if redact is None:
+        return value
+    with errors_as(AgentError, {Exception: lambda error: f"run.redact failed: {error}"}):
+        return redact(value)
+
+
+def _safe_cases(plan, cases) -> list[Case]:
+    return [replace(case, input=_redacted(plan.redact, case.input)) for case in cases]
+
+
+def _safe_failures(plan, cases, failures) -> list[tuple]:
+    # Inputs and error text are redacted before they reach the pattern and proposal calls.
+    inputs = {case.id: _redacted(plan.redact, case.input) for case in cases}
+    return [
+        (key, inputs[key], output, _redacted(plan.redact, reason))
+        for key, _, output, reason in failures
+    ]

promptcheckup/checkup/evaluate.py

@@ -0,0 +1,264 @@
+"""Shared paired evaluation: replay, score, decide, and conservatively downgrade."""
+
+import json
+import math
+from collections.abc import Sequence
+from dataclasses import asdict, dataclass, replace
+from statistics import fmean
+from typing import Protocol
+
+from promptcheckup.data import Case
+from promptcheckup.errors import errors_as, fallback
+from promptcheckup.llm import SelfCheck, judge_pairs, paired_scores, pairs_from_replay, self_check
+from promptcheckup.runner import AgentError, Replay, Run, prompt_hash, replay
+from promptcheckup.stats import Verdict, decide
+
+DOWNGRADES = (
+    (
+        lambda result: result.budget_exhausted,
+        lambda result: "call budget reached; raise run.max_calls or use fewer cases",
+    ),
+    (
+        lambda result: (
+            result.scores.self_check is not None and not result.scores.self_check.reliable
+        ),
+        lambda result: f"judge unreliable: {result.scores.self_check.reason}",
+    ),
+)
+JUDGE_CALLS = {"score": lambda n, s: 0, "judge": lambda n, s: 2 * n * (s + 1)}
+
+
+@dataclass(frozen=True)
+class Scores:
+    per_case: list[tuple[str, float]]
+    self_check: SelfCheck | None
+    skew_guard: bool
+    judge_calls: int = 0
+    prompt_tokens: int | None = None
+    completion_tokens: int | None = None
+    model: str = ""
+    budget_exhausted: bool = False
+
+
+class Scorer(Protocol):
+    """Turn a replay into per-case differences, spending at most `budget` further calls."""
+
+    def __call__(
+        self, cases: Sequence[Case], replay: Replay, *, budget: int, min_pairs: int
+    ) -> Scores: ...
+
+
+@dataclass(frozen=True)
+class Evaluation:
+    verdict: Verdict
+    scores: Scores
+    replay: Replay
+    errors: list[Run]
+    budget_exhausted: bool
+
+
+def score_mode(score_fn) -> Scorer:
+    """Average B-minus-A scores only over complete, successful sample pairs."""
+
+    def score(cases, replay, *, budget, min_pairs):
+        items = pairs_from_replay(cases, replay)
+        differences = [(key, _difference(score_fn, data, a, b)) for key, _, data, a, b in items]
+        return Scores(_per_case(differences), None, True)
+
+    return score
+
+
+def _difference(score_fn, case_input, a, b) -> float:
+    with errors_as(AgentError, {Exception: lambda error: f"scoring.score failed: {error}"}):
+        difference = float(score_fn(case_input, b)) - float(score_fn(case_input, a))
+        if not math.isfinite(difference):
+            raise ValueError("paired differences must be finite numbers")
+        return difference
+
+
+def _per_case(differences) -> list[tuple[str, float]]:
+    grouped = {}
+    for key, difference in differences:
+        grouped.setdefault(key, []).append(difference)
+    return [(key, fmean(values)) for key, values in grouped.items()]
+
+
+def judge_mode(llm, criteria, redact=None, *, cache=None) -> Scorer:
+    """Use swapped comparisons and a self-check, optionally cached for a named model."""
+
+    def score(cases, replay, *, budget, min_pairs):
+        safe_cases, safe_replay = _redacted(cases, replay, redact)
+        before = _tokens(llm)
+        scores = _judged(llm, criteria, cache, safe_cases, safe_replay, budget, min_pairs)
+        return replace(
+            scores, model=getattr(llm, "model", ""), **_token_delta(before, _tokens(llm))
+        )
+
+    return score
+
+
+def redacted_inputs(scorer: Scorer, redact) -> Scorer:
+    """Redact case inputs before scoring; agent outputs are redacted when they are produced."""
+    if redact is None:
+        return scorer
+
+    def score(cases, replay, *, budget, min_pairs):
+        with errors_as(AgentError, {Exception: lambda error: f"run.redact failed: {error}"}):
+            safe = [replace(case, input=redact(case.input)) for case in cases]
+        return scorer(safe, replay, budget=budget, min_pairs=min_pairs)
+
+    return score
+
+
+def _judged(llm, criteria, cache, cases, replay, budget, min_pairs) -> Scores:
+    items = pairs_from_replay(cases, replay)
+    limited = items[: budget // 2]  # each pair is judged in both orders
+    comparisons = judge_pairs(llm, limited, criteria)
+    ids = list(dict.fromkeys(item.case_id for item in comparisons))
+    values = list(zip(ids, paired_scores(comparisons), strict=True))
+    remaining = budget - 2 * len(limited)
+    check = _judge_self_check(llm, criteria, cache, cases, replay, remaining, min_pairs)
+    return Scores(
+        values,
+        check,
+        False,
+        judge_calls=2 * len(limited) + check.calls,
+        budget_exhausted=len(limited) < len(items) or check.budget_exhausted,
+    )
+
+
+def _judge_self_check(llm, criteria, cache, cases, replay, remaining, min_pairs):
+    key = _check_key(llm, criteria, cases, replay, min_pairs)
+    cached = _read_check(cache, key)
+    if cached is not None:
+        return replace(cached, calls=0)
+    check = self_check(llm, cases, replay, criteria, min_pairs=min_pairs, max_calls=remaining)
+    if cache is not None and key and not check.budget_exhausted:
+        cache.put(key, "judge-self-check", 0, asdict(check))
+    return check
+
+
+def _check_key(llm, criteria, cases, replay, min_pairs):
+    # The verdict depends on who judges, against what, and the A outputs judged.
+    judge = {"model": getattr(llm, "model", ""), "base_url": getattr(llm, "base_url", "")}
+    if not judge["model"]:
+        return ""  # an unnamed judge cannot be recognised next time
+    evidence = {
+        **judge,
+        "criteria": criteria,
+        "min_pairs": min_pairs,
+        "cases": [(case.id, case.input) for case in cases],
+        "runs": [
+            (run.case_id, run.sample, run.output, run.error)
+            for run in replay.runs
+            if run.prompt_set == "A"
+        ],
+    }
+    return prompt_hash({"self-check": json.dumps(evidence, sort_keys=True)})
+
+
+def _read_check(cache, key):
+    if cache is None or not key:
+        return None
+    found, payload = cache.get(key, "judge-self-check", 0)
+    if not found:
+        return None
+    return fallback(
+        lambda: _decoded_check(payload), lambda error: None, KeyError, TypeError, ValueError
+    )
+
+
+def _decoded_check(payload):
+    data = dict(payload)
+    verdict = data.pop("verdict")
+    check = SelfCheck(verdict=Verdict(**verdict) if verdict is not None else None, **data)
+    if type(check.reliable) is not bool or not 0 <= check.consistency <= 1:
+        raise ValueError("damaged self-check cache")
+    if check.reliable and check.verdict is None:
+        raise ValueError("reliable self-check needs a verdict")
+    return check
+
+
+def _tokens(llm) -> dict:
+    usage = getattr(llm, "usage", None)
+    return {name: getattr(usage, name, None) for name in ("prompt_tokens", "completion_tokens")}
+
+
+def _token_delta(before, after) -> dict:
+    return {name: None if before[name] is None else after[name] - before[name] for name in before}
+
+
+def _redacted(cases, replay, redact):
+    if redact is None:
+        return cases, replay
+    with errors_as(AgentError, {Exception: lambda error: f"run.redact failed: {error}"}):
+        safe_cases = [replace(case, input=redact(case.input)) for case in cases]
+        runs = [replace(run, output=redact(run.output)) for run in replay.runs if run.error is None]
+    return safe_cases, replace(replay, runs=runs)
+
+
+def evaluate(
+    agent,
+    cases,
+    prompts_a,
+    prompts_b,
+    scorer,
+    *,
+    samples,
+    timeout_s,
+    max_calls,
+    min_pairs,
+    cache,
+    confidence=0.95,
+) -> Evaluation:
+    """Evaluate paired evidence, preserving statistics even when a safeguard blocks it."""
+    replayed = replay(
+        agent,
+        cases,
+        prompts_a,
+        prompts_b,
+        samples=samples,
+        timeout_s=timeout_s,
+        max_calls=max_calls,
+        cache=cache,
+    )
+    return evaluate_replay(
+        cases,
+        replayed,
+        scorer,
+        budget=max_calls - replayed.calls,
+        min_pairs=min_pairs,
+        confidence=confidence,
+    )
+
+
+def evaluate_replay(cases, replayed, scorer, *, budget, min_pairs, confidence=0.95) -> Evaluation:
+    """Score existing evidence using the same decision and downgrade rules as evaluate."""
+    scores = scorer(cases, replayed, budget=budget, min_pairs=min_pairs)
+    verdict = decide(
+        [value for _, value in scores.per_case],
+        min_pairs=min_pairs,
+        skew_guard=scores.skew_guard,
+        confidence=confidence,
+    )
+    result = Evaluation(
+        verdict,
+        scores,
+        replayed,
+        [run for run in replayed.runs if run.error is not None],
+        replayed.budget_exhausted or scores.budget_exhausted,
+    )
+    return _downgraded(result)
+
+
+def _downgraded(result: Evaluation) -> Evaluation:
+    for predicate, reason in DOWNGRADES:
+        if predicate(result):
+            verdict = replace(result.verdict, decision="inconclusive", reason=reason(result))
+            return replace(result, verdict=verdict)
+    return result
+
+
+def planned_calls(n_cases, samples, mode) -> dict:
+    """Cold-cache agent calls and judge calls, including swaps and the A-vs-A check."""
+    return {"agent": 2 * n_cases * samples, "judge": JUDGE_CALLS[mode](n_cases, samples)}