copela 0.1.0__py3-none-any.whl

copela/__init__.py

@@ -0,0 +1,61 @@
+"""copela: run narrative-to-formal translation across many models, and score it with oracles
+that are not language models.
+
+The name is the cupel used in fire assay, the vessel that separates the metal from the lead. That is
+the job here: separating a formalization that is faithful from one that merely runs.
+
+Four layers, reported separately and never merged into one score:
+
+1. executable, did it run, solve, compile
+2. structural, is it the same model as the reference
+3. property, do the invariants of this class hold
+4. judge, what a model says, recorded as a labelled screening aggregate and never as truth
+
+The headline is the subtraction: how often the artifact ran, minus how often it was right.
+"""
+
+from __future__ import annotations
+
+from .budget import Budget, BudgetExceeded, estimate
+from .ledger import CallKey, Ledger, LedgerError, Record
+from .providers import Provider, ProviderError, StubProvider
+from .report import Cell, Report, build
+from .sweep import Case, Sweep, Target
+from .verdicts import (
+    JUDGE_LABEL,
+    CandidateVerdict,
+    Layer,
+    LayerResult,
+    Outcome,
+    Rate,
+)
+
+__version__ = "0.1.0"
+__display_version__ = "0.01.000"
+
+__all__ = [
+    "JUDGE_LABEL",
+    "Budget",
+    "BudgetExceeded",
+    "CallKey",
+    "CandidateVerdict",
+    "Case",
+    "Cell",
+    "Layer",
+    "LayerResult",
+    "Ledger",
+    "LedgerError",
+    "Outcome",
+    "Provider",
+    "ProviderError",
+    "Rate",
+    "Record",
+    "Report",
+    "StubProvider",
+    "Sweep",
+    "Target",
+    "__display_version__",
+    "__version__",
+    "build",
+    "estimate",
+]

copela/budget.py

@@ -0,0 +1,84 @@
+"""The budget guard: stop before the limit, not after it.
+
+A sweep is cases times models times repeats, and each cell costs money. The guard exists because the
+failure it prevents has happened on this account: an unattended job consumed a week of quota in
+about a day.
+
+The rule is that the guard refuses the call that *would* exceed the budget, rather than noticing
+afterwards. A guard that reports an overrun is an accountant, not a guard.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+class BudgetExceeded(RuntimeError):
+    """Raised when a call would take the sweep past its declared budget."""
+
+
+@dataclass
+class Budget:
+    """A spend ceiling and a kill criterion, both declared before the sweep runs.
+
+    ``limit_usd`` is the hard ceiling. ``max_consecutive_failures`` is the kill criterion: a sweep
+    whose calls are all failing is buying nothing, and continuing to the ceiling is waste.
+    """
+
+    limit_usd: float
+    max_consecutive_failures: int = 10
+    spent_usd: float = 0.0
+    consecutive_failures: int = 0
+    calls: int = 0
+
+    def __post_init__(self) -> None:
+        if self.limit_usd < 0:
+            raise ValueError("a budget cannot be negative")
+
+    @property
+    def remaining_usd(self) -> float:
+        return max(0.0, self.limit_usd - self.spent_usd)
+
+    def check(self, estimated_usd: float) -> None:
+        """Raise if this call would exceed the ceiling. Call BEFORE spending."""
+        if self.spent_usd + estimated_usd > self.limit_usd:
+            raise BudgetExceeded(
+                f"this call is estimated at {estimated_usd:.4f} USD and "
+                f"{self.spent_usd:.4f} of {self.limit_usd:.4f} is already spent; "
+                "stopping before the budget rather than after it"
+            )
+        if self.consecutive_failures >= self.max_consecutive_failures:
+            raise BudgetExceeded(
+                f"{self.consecutive_failures} consecutive failures reached the kill criterion; "
+                "a sweep that is failing every call is buying nothing"
+            )
+
+    def charge(self, actual_usd: float, *, failed: bool = False) -> None:
+        """Record what a completed call actually cost."""
+        self.spent_usd += actual_usd
+        self.calls += 1
+        self.consecutive_failures = self.consecutive_failures + 1 if failed else 0
+
+    def describe(self) -> str:
+        return (
+            f"{self.spent_usd:.4f} of {self.limit_usd:.4f} USD over {self.calls} call(s), "
+            f"{self.remaining_usd:.4f} remaining"
+        )
+
+
+def estimate(
+    prompt: str,
+    expected_output_tokens: int,
+    input_per_mtok: float,
+    output_per_mtok: float,
+) -> float:
+    """A cost estimate before the call, from a crude token count.
+
+    Four characters per token is a rough English average and it is deliberately not refined: the
+    estimate exists to keep the guard conservative, and a guard that under-estimates is worse than
+    one that stops slightly early.
+    """
+    input_tokens = max(1, len(prompt) // 4)
+    return (
+        input_tokens * input_per_mtok + expected_output_tokens * output_per_mtok
+    ) / 1_000_000

copela/cli.py

@@ -0,0 +1,158 @@
+"""The command line.
+
+Four commands, and the only one that spends money refuses to start without a declared budget.
+
+    copela models                     what each provider can serve, and what it costs
+    copela solve  <problem.json>      solve one formalization, no model involved
+    copela sweep  <cases.json> ...    run the sweep
+    copela report <ledger.jsonl>      the gap, from a ledger
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from . import __display_version__
+from .budget import Budget
+from .ledger import Ledger
+from .providers import ProviderError, get
+
+
+def _selectable_providers() -> list[str]:
+    """Every registered provider except the scripted stub, which is for dry runs."""
+    from .providers import REGISTRY
+
+    return [name for name in sorted(REGISTRY) if name != "stub"]
+
+
+def _cmd_models(args: argparse.Namespace) -> int:
+    for name in args.providers:
+        try:
+            provider = get(name)
+            models = provider.models()
+        except ProviderError as error:
+            print(f"{name}: unavailable ({error})")
+            continue
+        print(f"{name}:")
+        for model_id, pricing in sorted(models.items()):
+            if pricing.input_per_mtok or pricing.output_per_mtok:
+                cost = f"{pricing.input_per_mtok:g} in / {pricing.output_per_mtok:g} out per MTok"
+            else:
+                cost = "no per-token price (local)"
+            print(f"  {model_id:<40} {cost}")
+    return 0
+
+
+def _cmd_solve(args: argparse.Namespace) -> int:
+    from planteo import Problem, validate
+
+    problem = Problem.from_json(json.loads(Path(args.problem).read_text(encoding="utf-8")))
+    report = validate(problem)
+    if not report.ok:
+        print(report)
+        return 1
+
+    from .solvers.highs import SolverUnavailable, solve
+
+    try:
+        solution = solve(problem, solver_name=args.solver)
+    except SolverUnavailable as error:
+        print(error)
+        return 2
+
+    print(f"status: {solution.detail}")
+    if solution.objective is not None:
+        print(f"objective: {solution.objective:.6g}")
+    for name, value in sorted(solution.values.items()):
+        print(f"  {name} = {value:.6g}")
+    return 0 if solution.feasible else 1
+
+
+def _cmd_sweep(args: argparse.Namespace) -> int:
+    # A sweep is the only command that spends. It refuses to start without a budget rather than
+    # defaulting to one, because a default budget is a number nobody chose.
+    if args.budget_usd is None:
+        print(
+            "a sweep needs a declared budget: pass --budget-usd. "
+            "Every sweep states its ceiling and its kill criterion before it runs",
+            file=sys.stderr,
+        )
+        return 2
+
+    print(
+        f"sweep configured: budget {args.budget_usd} USD, "
+        f"{args.repeats} repeat(s), ledger {args.ledger}"
+    )
+    print(
+        "Assembling cases and targets is done from Python: the prompt strategy and the response "
+        "parser are what a study varies, so they are arguments rather than flags. "
+        "See docs/guides/01_run_a_sweep.md"
+    )
+    Budget(limit_usd=args.budget_usd)  # validates the ceiling now rather than mid-run
+    return 0
+
+
+def _cmd_report(args: argparse.Namespace) -> int:
+    from .report import build
+
+    ledger = Ledger(args.ledger)
+    if not Path(args.ledger).exists():
+        print(f"no ledger at {args.ledger}", file=sys.stderr)
+        return 2
+
+    report = build(ledger)
+    if args.json:
+        print(json.dumps(report.to_json(), indent=2))
+    else:
+        print(report.to_text())
+        print(f"\nledger: {len(ledger.records())} call(s), {ledger.total_cost_usd:.4f} USD")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="copela",
+        description=(
+            "Run narrative-to-formal translation across many models and score it with oracles "
+            "that are not language models."
+        ),
+    )
+    parser.add_argument("--version", action="version", version=f"copela {__display_version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    models = subparsers.add_parser("models", help="list what each provider can serve")
+    models.add_argument(
+        "providers",
+        nargs="*",
+        # Read from the registry rather than written out here. A hardcoded list would put vendor
+        # names outside the seam, which R-006 forbids and a test enforces.
+        default=_selectable_providers(),
+        help="provider names",
+    )
+    models.set_defaults(func=_cmd_models)
+
+    solve = subparsers.add_parser("solve", help="solve one formalization, no model involved")
+    solve.add_argument("problem", help="a planteo Problem as JSON")
+    solve.add_argument("--solver", default="appsi_highs")
+    solve.set_defaults(func=_cmd_solve)
+
+    sweep = subparsers.add_parser("sweep", help="run a sweep")
+    sweep.add_argument("--ledger", default="runs.jsonl")
+    sweep.add_argument("--budget-usd", type=float, default=None, help="required; the hard ceiling")
+    sweep.add_argument("--repeats", type=int, default=3)
+    sweep.set_defaults(func=_cmd_sweep)
+
+    report = subparsers.add_parser("report", help="the gap, from a ledger")
+    report.add_argument("ledger")
+    report.add_argument("--json", action="store_true")
+    report.set_defaults(func=_cmd_report)
+
+    args = parser.parse_args(argv)
+    return int(args.func(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

copela/ledger.py

@@ -0,0 +1,259 @@
+"""The run ledger: append-only JSONL, one record per call.
+
+Two properties, both enforced rather than intended.
+
+**Append-only.** A record that has been written is never modified. A ledger that can be rewritten is
+not evidence, and the whole value of this file is that a number in a report can be traced back to
+the call that produced it.
+
+**Every call pins its provenance.** Model id, model version, temperature, seed, provider
+fingerprint, repeat index and prompt digest, on every record. Without those a rate is a number with
+no subject, and six months later nobody can say which model produced it.
+
+The ledger is also the resume mechanism: a sweep reads it, sees which (case, model, repeat) triples
+are already done, and skips them. A long sweep that lost its work to an interruption is a cost this
+account has already paid once.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+LEDGER_SCHEMA = "copela-ledger/1.0"
+
+
+class LedgerError(RuntimeError):
+    """Raised when an operation would violate the append-only contract."""
+
+
+def digest(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:32]
+
+
+@dataclass(frozen=True, slots=True)
+class CallKey:
+    """What makes a call unique, and therefore what resume matches on."""
+
+    case_id: str
+    provider: str
+    model_id: str
+    repeat: int
+
+    def as_tuple(self) -> tuple[str, str, str, int]:
+        return (self.case_id, self.provider, self.model_id, self.repeat)
+
+
+@dataclass(frozen=True, slots=True)
+class Record:
+    """One call. Every field below is required; that is the point of R-003."""
+
+    key: CallKey
+    family: str
+    model_version: str
+    temperature: float
+    seed: int | None
+    provider_fingerprint: str
+    prompt_digest: str
+    response_digest: str
+    latency_ms: float
+    input_tokens: int
+    output_tokens: int
+    cost_usd: float
+    verdicts: list[dict[str, object]] = field(default_factory=list)
+    error: str = ""
+    #: A bounded excerpt of the raw response, kept ONLY when the call failed.
+    #:
+    #: A digest proves a response existed and says nothing about what was wrong with it. Diagnosing
+    #: a failure from a digest is impossible, and re-running to reproduce does not work either:
+    #: hosted inference is not deterministic, so the failure may not come back. The excerpt is
+    #: bounded because a ledger is evidence, not a transcript archive, and it is kept only on
+    #: failure because a successful run is already described by its verdicts.
+    response_excerpt: str = ""
+    recorded_at: str = ""
+
+    def to_json(self) -> dict[str, object]:
+        data = {
+            "schema": LEDGER_SCHEMA,
+            "case_id": self.key.case_id,
+            "provider": self.key.provider,
+            "model_id": self.key.model_id,
+            "repeat": self.key.repeat,
+            "family": self.family,
+            "model_version": self.model_version,
+            "temperature": self.temperature,
+            "seed": self.seed,
+            "provider_fingerprint": self.provider_fingerprint,
+            "prompt_digest": self.prompt_digest,
+            "response_digest": self.response_digest,
+            "latency_ms": self.latency_ms,
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "cost_usd": self.cost_usd,
+            "verdicts": self.verdicts,
+            "error": self.error,
+            "response_excerpt": self.response_excerpt,
+            "recorded_at": self.recorded_at
+            or datetime.now(UTC).isoformat(timespec="seconds"),
+        }
+        return data
+
+    @classmethod
+    def from_json(cls, data: dict[str, object]) -> Record:
+        return cls(
+            key=CallKey(
+                case_id=str(data["case_id"]),
+                provider=str(data["provider"]),
+                model_id=str(data["model_id"]),
+                repeat=int(data["repeat"]),  # type: ignore[arg-type]
+            ),
+            family=str(data.get("family", "")),
+            model_version=str(data.get("model_version", "")),
+            temperature=float(data.get("temperature", 0.0)),  # type: ignore[arg-type]
+            seed=None if data.get("seed") is None else int(data["seed"]),  # type: ignore[arg-type]
+            provider_fingerprint=str(data.get("provider_fingerprint", "")),
+            prompt_digest=str(data.get("prompt_digest", "")),
+            response_digest=str(data.get("response_digest", "")),
+            latency_ms=float(data.get("latency_ms", 0.0)),  # type: ignore[arg-type]
+            input_tokens=int(data.get("input_tokens", 0)),  # type: ignore[arg-type]
+            output_tokens=int(data.get("output_tokens", 0)),  # type: ignore[arg-type]
+            cost_usd=float(data.get("cost_usd", 0.0)),  # type: ignore[arg-type]
+            verdicts=list(data.get("verdicts", [])),  # type: ignore[arg-type]
+            error=str(data.get("error", "")),
+            response_excerpt=str(data.get("response_excerpt", "")),
+            recorded_at=str(data.get("recorded_at", "")),
+        )
+
+
+#: Fields whose absence makes a record useless as evidence. R-003.
+REQUIRED_PROVENANCE = (
+    "model_id",
+    "model_version",
+    "temperature",
+    "provider_fingerprint",
+    "prompt_digest",
+    "repeat",
+)
+
+
+class LedgerBusy(LedgerError):
+    """Another process holds this ledger."""
+
+
+class Ledger:
+    """An append-only JSONL file of call records.
+
+    ``exclusive=True`` takes a lock for the lifetime of the object, so two sweeps cannot write one
+    ledger. That is not hypothetical: a sweep was started while an earlier one was still alive, both
+    appended to the same file, and the result interleaved records from two different versions of the
+    code. Append-only does not help there, because the two processes write different keys, so
+    nothing collides and nothing complains. The file simply stops meaning one thing.
+    """
+
+    def __init__(self, path: str | os.PathLike[str], exclusive: bool = False) -> None:
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock_path = self.path.with_suffix(self.path.suffix + ".lock")
+        self._holds_lock = False
+        if exclusive:
+            self._acquire()
+
+    def _acquire(self) -> None:
+        try:
+            # O_EXCL is the whole mechanism: creating the file IS the lock, atomically.
+            descriptor = os.open(self._lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+        except FileExistsError:
+            holder = ""
+            try:
+                holder = self._lock_path.read_text(encoding="utf-8").strip()
+            except OSError:
+                pass
+            raise LedgerBusy(
+                f"{self.path} is locked by {holder or 'another process'}. Two sweeps writing one "
+                f"ledger interleave their records and the file stops meaning one thing. "
+                f"Stop the other run, or delete {self._lock_path.name} if it is stale"
+            ) from None
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(
+                f"pid {os.getpid()} since "
+                f"{datetime.now(UTC).isoformat(timespec='seconds')}\n"
+            )
+        self._holds_lock = True
+
+    def release(self) -> None:
+        if self._holds_lock:
+            self._lock_path.unlink(missing_ok=True)
+            self._holds_lock = False
+
+    def __enter__(self) -> Ledger:
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.release()
+
+    # -- writing ---------------------------------------------------------------------
+
+    def append(self, record: Record) -> None:
+        """Append one record. Refuses a duplicate key, because that would be a rewrite."""
+        payload = record.to_json()
+        missing = [
+            name
+            for name in REQUIRED_PROVENANCE
+            if payload.get(name) in (None, "")
+        ]
+        if missing:
+            raise LedgerError(
+                f"refusing to record a call missing its provenance: {', '.join(missing)}. "
+                "A rate over records that cannot say which model produced them is not evidence"
+            )
+        if self.has(record.key):
+            raise LedgerError(
+                f"{record.key.as_tuple()} is already in the ledger. The ledger is append-only; "
+                "a run is never edited, because a ledger that can be rewritten is not evidence"
+            )
+        with self.path.open("a", encoding="utf-8", newline="\n") as handle:
+            handle.write(json.dumps(payload, sort_keys=True) + "\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+
+    # -- reading ---------------------------------------------------------------------
+
+    def __iter__(self) -> Iterator[Record]:
+        if not self.path.exists():
+            return iter(())
+        return self._read()
+
+    def _read(self) -> Iterator[Record]:
+        with self.path.open("r", encoding="utf-8") as handle:
+            for number, line in enumerate(handle, start=1):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    yield Record.from_json(json.loads(line))
+                except (json.JSONDecodeError, KeyError) as error:
+                    raise LedgerError(
+                        f"{self.path}:{number} is not a readable record: {error}"
+                    ) from error
+
+    def records(self) -> list[Record]:
+        return list(self)
+
+    def keys(self) -> set[tuple[str, str, str, int]]:
+        return {record.key.as_tuple() for record in self}
+
+    def has(self, key: CallKey) -> bool:
+        return key.as_tuple() in self.keys()
+
+    def completed(self) -> set[tuple[str, str, str, int]]:
+        """Keys already done, so a resumed sweep can skip them. R-012."""
+        return self.keys()
+
+    @property
+    def total_cost_usd(self) -> float:
+        return sum(record.cost_usd for record in self)

copela/oracles/__init__.py

@@ -0,0 +1 @@
+"""Oracles: the layers that are not language models."""