puremethod 0.1.0__py3-none-any.whl

ledger/model.py

@@ -0,0 +1,107 @@
+"""Entries, derived views and reports (04-ledger.md §3, §4)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+Actor = Literal["loop", "user", "system"]
+
+ENTRY_TYPES: tuple[str, ...] = (
+    "run.created",
+    "run.status",
+    "checkpoint.requested",
+    "checkpoint.decided",
+    "call.recorded",
+    "idea.proposed",
+    "hypothesis.proposed",
+    "hypothesis.novelty",
+    "screening.result",
+    "prereg.registered",
+    "prereg.amended",
+    "experiment.started",
+    "experiment.finished",
+    "decision.made",
+    "selfcheck.result",
+    "paper.built",
+    "correction",
+)
+
+ARTIFACT_KINDS: tuple[str, ...] = (
+    "code", "env", "stdout", "stderr", "result", "figure", "paper", "prompt", "response",
+    "other",
+)
+
+CONFIRM_STAGE = "confirm"
+SMOKE_STAGE = "smoke"  # harness checks; no decision uses them (INV-9, KB-018)
+
+
+@dataclass(frozen=True)
+class Entry:
+    run_id: str
+    seq: int
+    entry_id: str
+    type: str
+    at: str
+    actor: Actor
+    payload: dict[str, Any]
+    refs: list[str]
+    prev_hash: str
+    hash: str
+
+    def hashed_fields(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "seq": self.seq,
+            "entry_id": self.entry_id,
+            "type": self.type,
+            "at": self.at,
+            "actor": self.actor,
+            "payload": self.payload,
+            "refs": self.refs,
+            "prev_hash": self.prev_hash,
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        return {**self.hashed_fields(), "hash": self.hash}
+
+
+@dataclass(frozen=True)
+class Rejection:
+    run_id: str
+    at: str
+    target: str  # entry type, or "artifact"
+    rule: str
+    message: str
+
+
+@dataclass
+class RunState:
+    """Derived view: a pure function of the entries (04-ledger.md §5 step 2)."""
+
+    run_id: str
+    status: str | None = None
+    hypotheses: dict[str, dict[str, Any]] = field(default_factory=dict)
+    prereg_in_force: dict[tuple[str, str], str] = field(default_factory=dict)
+    open_experiments: list[str] = field(default_factory=list)
+    finished: dict[str, str] = field(default_factory=dict)
+    decisions: list[str] = field(default_factory=list)
+    last_seq: int = 0
+
+
+@dataclass(frozen=True)
+class SpendSummary:
+    total_usd: float
+    by_source: dict[str, float]
+    unmetered_items: int
+    money_limit_usd: float | None
+
+
+@dataclass(frozen=True)
+class ChainReport:
+    run_id: str
+    entries: int
+    ok: bool
+    first_break_seq: int | None = None
+    reason: str | None = None
+    head_hash: str | None = None

ledger/payloads.py

@@ -0,0 +1,228 @@
+"""Payload schemas per entry type (04-ledger.md §4, "Entry types (v1)").
+
+The KB lists the main fields of each type; extra fields are allowed. Fields that hold entry
+ids are checked by the rules in `rules.py`.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from .errors import LedgerRejection
+
+NonEmpty = Field(min_length=1)
+
+REASON_CODES = (
+    "refuted", "harness_error", "not_novel", "infeasible", "no_signal", "superseded",
+    "user_stopped", "limit_reached",
+)
+ReasonCode = Literal[
+    "refuted", "harness_error", "not_novel", "infeasible", "no_signal", "superseded",
+    "user_stopped", "limit_reached",
+]
+DataRole = Literal["selection", "held_out"]
+
+
+class _Payload(BaseModel):
+    model_config = ConfigDict(extra="allow", strict=True)
+
+
+class RunCreated(_Payload):
+    idea: str = NonEmpty
+    config: dict[str, Any]
+
+
+class RunStatus(_Payload):
+    status: Literal["preflight", "running", "paused", "stopping", "writing", "delivered",
+                    "failed"]
+    reason: str = ""
+
+
+class CheckpointRequested(_Payload):
+    checkpoint_id: str = NonEmpty
+    options: Any = None
+
+
+class CheckpointDecided(_Payload):
+    checkpoint_id: str = NonEmpty
+    decision: Any
+
+
+class CallRecorded(_Payload):
+    source: Literal["llm", "purescholar", "puretrace"]
+    record: dict[str, Any]
+    cost_usd: float | None
+    metered: bool
+    request_artifact: str | None = None
+    response_artifact: str | None = None
+
+
+class IdeaProposed(_Payload):
+    direction: str = NonEmpty
+    parents: list[str] = []
+
+
+class HypothesisProposed(_Payload):
+    statement: str = NonEmpty
+    idea: str = NonEmpty
+    kind: Literal["treatment", "baseline"]
+
+
+class HypothesisNovelty(_Payload):
+    hypothesis: str = NonEmpty
+    evidence: list[str]
+    judgement: Literal["novel", "overlap", "unclear"]
+    reason: str = NonEmpty
+
+
+class ScreeningResult(_Payload):
+    paper: Any
+    assessment: dict[str, Any]
+    decision: str = NonEmpty
+
+
+class Metric(_Payload):
+    name: str = NonEmpty
+    unit: str
+    direction: Literal["maximize", "minimize"]
+
+
+class Comparison(_Payload):
+    baseline: str = NonEmpty
+    rule: str = NonEmpty
+    threshold: float
+
+
+class DataSet(_Payload):
+    source: str = NonEmpty
+    licence: str | None = None
+    role: DataRole
+    digest: str = NonEmpty
+
+
+class Budget(_Payload):
+    wall_clock_s: float = Field(gt=0)
+    compute: Any = None
+    money_usd_estimate: float = Field(ge=0)
+
+
+class Seeds(_Payload):
+    policy: str = NonEmpty
+    values: list[int] = Field(min_length=1)
+
+
+class PreregRegistered(_Payload):
+    hypothesis: str = NonEmpty
+    stage: str = NonEmpty
+    metric: Metric
+    comparison: Comparison
+    evaluation_data: list[DataSet] = Field(min_length=1)
+    budget: Budget
+    seeds: Seeds
+    code_digest: str = NonEmpty
+
+
+class PreregAmended(_Payload):
+    supersedes: str = NonEmpty
+    changes: dict[str, Any] = Field(min_length=1)
+    reason: str = NonEmpty
+
+
+class ExperimentStarted(_Payload):
+    prereg: str = NonEmpty
+    executor_id: str = NonEmpty
+    isolation: Literal["container", "vm", "none"]
+    code_digest: str = NonEmpty
+    env_digest: str = NonEmpty
+    seed: int
+
+
+class MetricValue(_Payload):
+    name: str = NonEmpty
+    value: float
+    unit: str
+    data_role: DataRole
+    set: str | None = None
+
+
+class ExperimentFinished(_Payload):
+    experiment: str = NonEmpty
+    status: Literal["ok", "failed", "crashed", "timeout", "killed", "lost"]
+    metrics: list[MetricValue] = []
+    artifacts: dict[str, str] = {}
+    duration_s: float | None = None
+    cost_usd: float | None = None
+    metered: bool = False
+
+
+class MetricUse(_Payload):
+    experiment: str = NonEmpty  # an experiment.finished entry id
+    name: str = NonEmpty
+    set: str | None = None
+
+
+class DecisionMade(_Payload):
+    hypothesis: str = NonEmpty
+    stage: str = NonEmpty
+    outcome: Literal["promote", "discard", "confirm", "inconclusive"]
+    reason_code: ReasonCode | None = None
+    reason_text: str = NonEmpty
+    prereg: str | None = None
+    experiments: list[str] = []  # experiment.started entry ids
+    metrics_used: list[MetricUse] = []
+    comparison: dict[str, Any] = {}
+
+
+class SelfcheckResult(_Payload):
+    paper: str = NonEmpty
+    assessment: dict[str, Any]
+
+
+class PaperBuilt(_Payload):
+    artifacts: dict[str, str]
+    trace_map: dict[str, Any]
+    citation_keys: list[str]
+
+
+class Correction(_Payload):
+    corrects: str = NonEmpty
+    problem: str = NonEmpty
+    correct: Any
+
+
+SCHEMAS: dict[str, type[_Payload]] = {
+    "run.created": RunCreated,
+    "run.status": RunStatus,
+    "checkpoint.requested": CheckpointRequested,
+    "checkpoint.decided": CheckpointDecided,
+    "call.recorded": CallRecorded,
+    "idea.proposed": IdeaProposed,
+    "hypothesis.proposed": HypothesisProposed,
+    "hypothesis.novelty": HypothesisNovelty,
+    "screening.result": ScreeningResult,
+    "prereg.registered": PreregRegistered,
+    "prereg.amended": PreregAmended,
+    "experiment.started": ExperimentStarted,
+    "experiment.finished": ExperimentFinished,
+    "decision.made": DecisionMade,
+    "selfcheck.result": SelfcheckResult,
+    "paper.built": PaperBuilt,
+    "correction": Correction,
+}
+
+
+def validate_payload(entry_type: str, payload: dict[str, Any]) -> _Payload:
+    schema = SCHEMAS.get(entry_type)
+    if schema is None:
+        raise LedgerRejection("schema", f"unknown entry type {entry_type!r}")
+    try:
+        return schema.model_validate(payload)
+    except ValidationError as exc:
+        # Field locations and error kinds only: never echo input values (INV-7).
+        problems = "; ".join(
+            f"{'.'.join(str(p) for p in err['loc']) or '$'}: {err['type']}"
+            for err in exc.errors(include_input=False, include_url=False)
+        )
+        raise LedgerRejection("schema", f"{entry_type}: {problems}") from None

ledger/rules.py

@@ -0,0 +1,204 @@
+"""Replay index and write-time rules (04-ledger.md §5, §7 INV-3 to INV-6).
+
+`RunIndex` is built by replaying entries in order. The same replay serves `state()` and the
+rules, so what the rules check is exactly what a resumed run sees.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from . import payloads as p
+from .errors import LedgerRejection
+from .model import CONFIRM_STAGE, SMOKE_STAGE, Entry, RunState
+
+# Outcomes that need experiments, and reason codes that can stand without them
+# (for example a novelty discard before anything ran).
+_NEEDS_EXPERIMENTS = {"promote", "confirm"}
+_REASONS_WITHOUT_EXPERIMENTS = {"not_novel", "infeasible", "harness_error", "superseded",
+                                "user_stopped", "limit_reached"}
+
+
+@dataclass
+class RunIndex:
+    run_id: str
+    by_id: dict[str, Entry] = field(default_factory=dict)
+    status: str | None = None
+    hypotheses: dict[str, dict[str, Any]] = field(default_factory=dict)
+    prereg_key: dict[str, tuple[str, str]] = field(default_factory=dict)
+    in_force: dict[tuple[str, str], str] = field(default_factory=dict)
+    started_prereg: dict[str, str] = field(default_factory=dict)
+    finished_for: dict[str, str] = field(default_factory=dict)
+    finished_of: dict[str, str] = field(default_factory=dict)
+    decisions: list[str] = field(default_factory=list)
+    last_seq: int = 0
+    last_hash: str | None = None
+
+    def apply(self, e: Entry) -> None:
+        self.by_id[e.entry_id] = e
+        self.last_seq = e.seq
+        self.last_hash = e.hash
+        pl = e.payload
+        if e.type == "run.status":
+            self.status = pl["status"]
+        elif e.type == "hypothesis.proposed":
+            self.hypotheses[e.entry_id] = {"statement": pl["statement"], "kind": pl["kind"],
+                                           "idea": pl["idea"]}
+        elif e.type == "prereg.registered":
+            key = (pl["hypothesis"], pl["stage"])
+            self.prereg_key[e.entry_id] = key
+            self.in_force[key] = e.entry_id
+        elif e.type == "prereg.amended":
+            key = self.prereg_key[pl["supersedes"]]
+            self.prereg_key[e.entry_id] = key
+            self.in_force[key] = e.entry_id
+        elif e.type == "experiment.started":
+            self.started_prereg[e.entry_id] = pl["prereg"]
+        elif e.type == "experiment.finished":
+            self.finished_for[pl["experiment"]] = e.entry_id
+            self.finished_of[e.entry_id] = pl["experiment"]
+        elif e.type == "decision.made":
+            self.decisions.append(e.entry_id)
+
+    def open_experiments(self) -> list[str]:
+        return [s for s in self.started_prereg if s not in self.finished_for]
+
+    def to_state(self) -> RunState:
+        return RunState(
+            run_id=self.run_id, status=self.status, hypotheses=dict(self.hypotheses),
+            prereg_in_force=dict(self.in_force), open_experiments=self.open_experiments(),
+            finished=dict(self.finished_for), decisions=list(self.decisions),
+            last_seq=self.last_seq,
+        )
+
+
+def _require_ref(refs: list[str], entry_id: str, field_name: str) -> None:
+    if entry_id not in refs:
+        raise LedgerRejection("refs", f"{field_name} {entry_id!r} must also be listed in refs")
+
+
+def _require_type(idx: RunIndex, entry_id: str, types: tuple[str, ...], what: str) -> Entry:
+    e = idx.by_id.get(entry_id)
+    if e is None or e.type not in types:
+        raise LedgerRejection("refs", f"{what} {entry_id!r} is not a {' or '.join(types)} "
+                                      "entry of this run")
+    return e
+
+
+def check(idx: RunIndex, entry_type: str, model: p._Payload, refs: list[str]) -> None:
+    """Raise LedgerRejection if the new entry breaks a rule. Order: refs, then type rules."""
+    for ref in refs:
+        if ref not in idx.by_id:
+            raise LedgerRejection("refs", f"ref {ref!r} does not exist earlier in this run")
+
+    if entry_type == "run.created":
+        if idx.last_seq != 0:
+            raise LedgerRejection("run.created", "a run has exactly one run.created, first")
+        return
+    if idx.last_seq == 0:
+        raise LedgerRejection("run.created", "the first entry of a run must be run.created")
+
+    if isinstance(model, p.HypothesisProposed):
+        _require_ref(refs, model.idea, "idea")
+        _require_type(idx, model.idea, ("idea.proposed",), "idea")
+    elif isinstance(model, p.HypothesisNovelty):
+        _require_ref(refs, model.hypothesis, "hypothesis")
+        _require_type(idx, model.hypothesis, ("hypothesis.proposed",), "hypothesis")
+    elif isinstance(model, p.PreregRegistered):
+        _check_prereg(idx, model, refs)
+    elif isinstance(model, p.PreregAmended):
+        _require_ref(refs, model.supersedes, "supersedes")
+        _require_type(idx, model.supersedes, ("prereg.registered", "prereg.amended"),
+                      "supersedes")
+    elif isinstance(model, p.ExperimentStarted):
+        _check_started(idx, model, refs)  # INV-3
+    elif isinstance(model, p.ExperimentFinished):
+        _check_finished(idx, model, refs)  # INV-6
+    elif isinstance(model, p.DecisionMade):
+        _check_decision(idx, model, refs)  # INV-4, INV-5
+    elif isinstance(model, p.Correction):
+        _require_ref(refs, model.corrects, "corrects")
+
+
+def _check_prereg(idx: RunIndex, m: p.PreregRegistered, refs: list[str]) -> None:
+    _require_ref(refs, m.hypothesis, "hypothesis")
+    _require_type(idx, m.hypothesis, ("hypothesis.proposed",), "hypothesis")
+    _require_ref(refs, m.comparison.baseline, "comparison.baseline")
+    _require_type(idx, m.comparison.baseline, ("hypothesis.proposed",), "comparison.baseline")
+
+
+def _check_started(idx: RunIndex, m: p.ExperimentStarted, refs: list[str]) -> None:
+    _require_ref(refs, m.prereg, "prereg")
+    _require_type(idx, m.prereg, ("prereg.registered", "prereg.amended"), "prereg")
+    key = idx.prereg_key[m.prereg]
+    if idx.in_force.get(key) != m.prereg:
+        raise LedgerRejection("INV-3", f"prereg {m.prereg!r} is not the pre-registration in "
+                                       f"force for hypothesis {key[0]!r} at stage {key[1]!r}")
+
+
+def _check_finished(idx: RunIndex, m: p.ExperimentFinished, refs: list[str]) -> None:
+    _require_ref(refs, m.experiment, "experiment")
+    _require_type(idx, m.experiment, ("experiment.started",), "experiment")
+    if m.experiment in idx.finished_for:
+        raise LedgerRejection("INV-6", f"experiment {m.experiment!r} already has an "
+                                       "experiment.finished entry")
+
+
+def _check_decision(idx: RunIndex, m: p.DecisionMade, refs: list[str]) -> None:
+    if m.stage == SMOKE_STAGE:
+        raise LedgerRejection("INV-9", "no decision is made at stage `smoke` (harness checks)")
+    _require_ref(refs, m.hypothesis, "hypothesis")
+    _require_type(idx, m.hypothesis, ("hypothesis.proposed",), "hypothesis")
+
+    if m.outcome in {"discard", "inconclusive"} and m.reason_code is None:
+        raise LedgerRejection("reasons", f"a {m.outcome} decision needs a reason_code")
+
+    if not m.experiments:
+        if m.outcome in _NEEDS_EXPERIMENTS or m.reason_code not in _REASONS_WITHOUT_EXPERIMENTS:
+            raise LedgerRejection("INV-4", f"a {m.outcome} decision with reason "
+                                           f"{m.reason_code!r} must cite the experiments it used")
+        if m.metrics_used:
+            raise LedgerRejection("INV-4", "metrics_used without experiments")
+        return
+
+    # INV-4: cite the pre-registration in force at each start, and each finished entry.
+    if m.prereg is None:
+        raise LedgerRejection("INV-4", "a decision that uses experiments must name its prereg")
+    _require_ref(refs, m.prereg, "prereg")
+    key = idx.prereg_key.get(m.prereg)
+    if key is None:
+        raise LedgerRejection("INV-4", f"prereg {m.prereg!r} is not a pre-registration")
+    if key != (m.hypothesis, m.stage):
+        raise LedgerRejection("INV-4", "the prereg does not belong to this hypothesis and stage")
+    finished_ids: set[str] = set()
+    for started in m.experiments:
+        _require_type(idx, started, ("experiment.started",), "experiment")
+        if idx.started_prereg[started] != m.prereg:
+            raise LedgerRejection("INV-4", f"experiment {started!r} started under a different "
+                                           "pre-registration")
+        finished = idx.finished_for.get(started)
+        if finished is None:
+            raise LedgerRejection("INV-4", f"experiment {started!r} has not finished")
+        _require_ref(refs, finished, "finished experiment")
+        finished_ids.add(finished)
+
+    # A decision computed from metrics must say which, or INV-5 could not be checked.
+    if not m.metrics_used and (m.outcome in _NEEDS_EXPERIMENTS or m.reason_code == "refuted"):
+        raise LedgerRejection("INV-5", f"a {m.outcome} decision must list the metrics it used")
+
+    # INV-5 (KB-014): held-out metrics only in decisions at the confirmation stage.
+    for use in m.metrics_used:
+        if use.experiment not in finished_ids:
+            raise LedgerRejection("INV-4", f"metric source {use.experiment!r} is not a finished "
+                                           "experiment of this decision")
+        metrics = idx.by_id[use.experiment].payload.get("metrics", [])
+        match = [x for x in metrics if x["name"] == use.name
+                 and (use.set is None or x.get("set") == use.set)]
+        if not match:
+            raise LedgerRejection("INV-4", f"metric {use.name!r} is not recorded in "
+                                           f"{use.experiment!r}")
+        held_out = any(x["data_role"] == "held_out" for x in match)
+        if held_out and m.stage != CONFIRM_STAGE:
+            raise LedgerRejection("INV-5", f"a {m.outcome} decision at stage {m.stage!r} may "
+                                           "not use held-out metrics")