masterwork 0.0.1__py3-none-any.whl

masterwork/pairs.py

@@ -0,0 +1,252 @@
+"""Turn scored runs into training data — and refuse to turn noise into it.
+
+A battery already produces what preference training wants: the same scene
+attempted several times by the same agent, each attempt scored on the same
+axes by a judge that is not the agent. Pairing a run that scored well with
+one that scored badly gives an on-policy pair with an external label, which
+is the pair a hand-built set cannot be.
+
+This matters because hand-built sets fail in a specific way. A set written
+by the house teaches the behaviour the house imagined, and an audit of one
+such set found more than a third of its pairs teaching something other than
+the axis they were written for. Pairs cut from real runs cannot drift that
+way: whatever the model actually did is what gets rewarded or not.
+
+Two refusals are built in, both from measurements rather than taste:
+
+  * **No pair from a self-judged run.** If the agent scored itself, the
+    label is the agent's opinion of itself and training on it closes a loop
+    that has nothing outside it.
+  * **No pair from a gap smaller than the judge's own spread.** A judge that
+    moves by 0.15 between identical runs will happily label two equivalent
+    trajectories as winner and loser. Pairs built from that teach the model
+    to imitate the judge's noise. The threshold is a parameter because it is
+    a measurement, not a constant — measure it for your judge first.
+
+The system message can be stripped (`--strip-system`). Kept, the data
+teaches behaviour conditional on the identity being present; stripped, it
+teaches the behaviour itself, which is the point if the aim is to stop
+paying for the prompt on every request.
+"""
+from __future__ import annotations
+
+import argparse
+import glob
+import json
+import os
+import sys
+
+
+def run_is_self_judged(run_dir: str) -> bool | None:
+    """Read the benchmark's own stamp, from where it actually writes it.
+
+    The refusal used to look for `seal.judge == "SELF"` on each cell. The
+    benchmark never writes that: a cell's seal is built once, before judging,
+    and carries the agent's definition only — the self_judged stamp lives at
+    the top of report.json. So the check could not fire, and pairs cut from a
+    fully self-judged run came out looking like pairs cut from a judged one.
+
+    None means no report was found, which is not the same as False.
+    """
+    for p in sorted(glob.glob(os.path.join(run_dir, "**", "report.json"),
+                              recursive=True)):
+        try:
+            return bool(json.load(open(p, encoding="utf-8")).get("self_judged"))
+        except Exception:
+            continue
+    return None
+
+
+def load_cells(run_dir: str) -> list[dict]:
+    out = []
+    for p in sorted(glob.glob(os.path.join(run_dir, "**", "cells", "*.json"),
+                              recursive=True)):
+        try:
+            out.append(json.load(open(p, encoding="utf-8")))
+        except Exception:
+            continue
+    return out
+
+
+def axis_scores(cell: dict) -> dict[str, float]:
+    """Judged axes score 1 when the verdict is the positive label; counted
+    axes carry their number. The two arrive under different keys and must
+    not be flattened together — a counted fact dressed as a judgement is
+    exactly the confusion the report contract exists to prevent."""
+    scores: dict[str, float] = {}
+    for axis, v in (cell.get("verdicts") or {}).items():
+        if not isinstance(v, dict):
+            continue
+        verdict, positive = v.get("verdict"), v.get("positive")
+        if verdict in (None, "n/a", "na"):
+            if v.get("na_means") == "failure":
+                scores[axis] = 0.0
+            continue
+        if positive is not None:
+            scores[axis] = float(verdict == positive)
+    for axis, v in (cell.get("event_axes") or {}).items():
+        if isinstance(v, (int, float)):
+            scores[axis] = float(v)
+    return scores
+
+
+def usable(cell: dict, allow_self_judged: bool) -> str | None:
+    if cell.get("invalid"):
+        return f"invalid cell ({cell.get('invalid_reason')})"
+    if not cell.get("messages"):
+        return "no transcript"
+    return None
+
+
+def split_prompt(messages: list[dict], strip_system: bool):
+    """Prefix shared by both trajectories, and the trajectory itself.
+
+    Runs of the same scene diverge at the first assistant turn, so the shared
+    prefix is the scene as given. The preference is therefore over whole
+    trajectories, which is what is actually being preferred.
+    """
+    prefix, rest = [], []
+    for i, m in enumerate(messages):
+        if m.get("role") in ("system", "user") and not rest:
+            if m.get("role") == "system" and strip_system:
+                continue
+            prefix.append(m)
+        else:
+            rest = messages[i:]
+            break
+    return prefix, rest
+
+
+def build(cells: list[dict], axis: str, min_gap: float, strip_system: bool,
+          allow_self_judged: bool, gap_from: str | None = None
+          ) -> tuple[list[dict], list[str]]:
+    notes, by_scene = [], {}
+    for c in cells:
+        why = usable(c, allow_self_judged)
+        if why:
+            notes.append(f"skipped {c.get('cell_id')}: {why}")
+            continue
+        s = axis_scores(c).get(axis)
+        if s is None:
+            notes.append(f"skipped {c.get('cell_id')}: no {axis}")
+            continue
+        by_scene.setdefault(c.get("scene"), []).append((s, c))
+
+    pairs = []
+    for scene, scored in sorted(by_scene.items()):
+        scored.sort(key=lambda x: x[0], reverse=True)
+        used = set()
+        for i, (hi, top) in enumerate(scored):
+            for j in range(len(scored) - 1, i, -1):
+                lo, bottom = scored[j]
+                if j in used or hi - lo < min_gap:
+                    continue
+                prefix, chosen = split_prompt(top["messages"], strip_system)
+                _p, rejected = split_prompt(bottom["messages"], strip_system)
+                pairs.append({
+                    "axis": axis, "scene": scene,
+                    "prompt": prefix, "chosen": chosen, "rejected": rejected,
+                    "scores": {"chosen": hi, "rejected": lo, "gap": hi - lo},
+                    "provenance": {
+                        "gap_from": gap_from,
+                        "chosen_cell": top.get("cell_id"),
+                        "rejected_cell": bottom.get("cell_id"),
+                        "chosen_seed": top.get("seed"),
+                        "rejected_seed": bottom.get("seed"),
+                        "seal": top.get("seal"),
+                        "min_gap": min_gap,
+                    }})
+                used.add(j)
+                break
+    return pairs, notes
+
+
+def winners(cells: list[dict], axis: str, floor: float, strip_system: bool,
+            allow_self_judged: bool) -> list[dict]:
+    """Supervised set: the trajectories that scored at or above the floor.
+
+    The post-mortem of one preference round said to try this first — a
+    supervised pass moves the target directly, where a preference pass has
+    to move it against a divergence penalty that is there precisely to stop
+    large moves.
+    """
+    out = []
+    for c in cells:
+        if usable(c, allow_self_judged):
+            continue
+        s = axis_scores(c).get(axis)
+        if s is None or s < floor:
+            continue
+        prefix, rest = split_prompt(c["messages"], strip_system)
+        out.append({"axis": axis, "scene": c.get("scene"),
+                    "messages": prefix + rest, "score": s,
+                    "provenance": {"cell": c.get("cell_id"), "seed": c.get("seed"),
+                                   "seal": c.get("seal")}})
+    return out
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="masterwork pairs", description="cut training data from scored runs")
+    ap.add_argument("run_dir", help="a benchmark run directory (cells/ inside)")
+    ap.add_argument("--axis", required=True)
+    ap.add_argument("--out", required=True, help="JSONL destination")
+    ap.add_argument("--gap-from", metavar="FILE",
+                    help="the measurement the gap comes from — a file recording "
+                         "the judge's own spread. Recorded with every pair, so a "
+                         "set can be traced to the measurement that justified it")
+    ap.add_argument("--min-gap", type=float, required=True,
+                    help="required score difference. Measure your judge's own "
+                         "spread first and set this above it; a gap below it "
+                         "pairs the judge's noise, not the agent's behaviour")
+    ap.add_argument("--sft", action="store_true",
+                    help="winners only, as a supervised set")
+    ap.add_argument("--floor", type=float, default=1.0, help="--sft: minimum score")
+    ap.add_argument("--strip-system", action="store_true",
+                    help="drop the identity from the prompt: teach the behaviour "
+                         "itself rather than behaviour conditional on the prompt")
+    ap.add_argument("--allow-self-judged", action="store_true")
+    a = ap.parse_args(argv)
+
+    cells = load_cells(a.run_dir)
+    if not cells:
+        print(f"no cells under {a.run_dir}")
+        return 1
+
+    # A run-level fact, refused at run level. Cutting pairs from a run the
+    # benchmark itself marked incomparable trains the model on its own opinion
+    # of itself, and the resulting file looks exactly like a good one.
+    self_judged = run_is_self_judged(a.run_dir)
+    if self_judged and not a.allow_self_judged:
+        print(f"HELD: {a.run_dir} was judged by the agent's own endpoint "
+              f"(report.json says self_judged). The label would be the agent's "
+              f"opinion of itself. Pass --allow-self-judged to say you meant it, "
+              f"and record that you did.")
+        return 1
+    if self_judged is None:
+        print("  note: no report.json under this run directory, so the "
+              "self-judged stamp could not be read. It is not absent, it is "
+              "unchecked.")
+    if a.sft:
+        rows = winners(cells, a.axis, a.floor, a.strip_system, a.allow_self_judged)
+        notes = []
+    else:
+        rows, notes = build(cells, a.axis, a.min_gap, a.strip_system,
+                            a.allow_self_judged, a.gap_from)
+        if not a.gap_from:
+            print("  note: --gap-from not given. The gap is a measurement of "
+                  "your judge, not a setting; a set built from a number nobody "
+                  "measured is the quiet way to bake noise into weights.")
+    for n in notes:
+        print(f"  {n}")
+    with open(a.out, "w", encoding="utf-8") as f:
+        for r in rows:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+    kind = "examples" if a.sft else "pairs"
+    print(f"{len(rows)} {kind} from {len(cells)} cells -> {a.out}")
+    if not rows:
+        print("nothing met the bar; that is a result, not an error")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

masterwork/retain.py

@@ -0,0 +1,117 @@
+"""What survives a run, and what does not.
+
+A line that keeps everything becomes a disk problem, and then a slow one:
+every future search walks the transcripts of runs nobody will reopen. A
+line that keeps nothing cannot defend a number six weeks later. The split
+that works is not "keep less", it is: **heavy artefacts stay local and
+disposable, light evidence is archived on purpose.**
+
+Local and disposable — transcripts, judge logs, per-cell records, the
+benchmark's run directory. They live under a runs directory that version
+control ignores, and they are re-derivable: the seal names the corpus, the
+script and both seeds.
+
+Archived on purpose — the run record and the benchmark's report. Both are
+kilobytes. Together they carry the seal, the grid, the axes, the gate and
+the verdict, which is everything a later disagreement can be settled with.
+
+Excerpts are capped rather than dropped, so a record can quote without
+carrying. Three hundred characters is enough to recognise an answer and far
+too little to store one.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+
+EXCERPT = 300
+# What is worth copying out of a run directory when the run is over.
+ARCHIVE = ("run.json", "report.json")
+
+
+def excerpt(text: str | None, limit: int = EXCERPT) -> str:
+    text = (text or "").strip()
+    return text if len(text) <= limit else text[:limit] + "…"
+
+
+def dir_size(path: str) -> int:
+    total = 0
+    for root, _dirs, files in os.walk(path):
+        for f in files:
+            try:
+                total += os.path.getsize(os.path.join(root, f))
+            except OSError:
+                pass
+    return total
+
+
+def largest(path: str, n: int = 5) -> list[tuple[int, str]]:
+    found = []
+    for root, _dirs, files in os.walk(path):
+        for f in files:
+            p = os.path.join(root, f)
+            try:
+                found.append((os.path.getsize(p), p))
+            except OSError:
+                pass
+    return sorted(found, reverse=True)[:n]
+
+
+def check(path: str, max_bytes: int) -> list[str]:
+    """Report — do not delete. Deciding what to remove is not the line's call."""
+    size = dir_size(path)
+    if size <= max_bytes:
+        return []
+    out = [f"run directory is {size/1e6:.1f} MB, over the {max_bytes/1e6:.1f} MB "
+           f"mark — this is local and disposable, but say so before it grows a habit"]
+    out += [f"  {s/1e6:.1f} MB {p}" for s, p in largest(path)]
+    return out
+
+
+def archive(run_dir: str, dest: str) -> list[str]:
+    """Copy only the light, decision-bearing files out of a run directory."""
+    os.makedirs(dest, exist_ok=True)
+    copied = []
+    for root, _dirs, files in os.walk(run_dir):
+        for f in files:
+            if f in ARCHIVE:
+                src = os.path.join(root, f)
+                rel = os.path.relpath(src, run_dir).replace(os.sep, "-")
+                shutil.copy2(src, os.path.join(dest, rel))
+                copied.append(rel)
+    return copied
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="masterwork retain", description="run-directory retention")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    c = sub.add_parser("check", help="report size without deleting anything")
+    c.add_argument("run_dir")
+    c.add_argument("--max-mb", type=float, default=50.0)
+    a2 = sub.add_parser("archive", help="copy out the kilobyte-sized evidence")
+    a2.add_argument("run_dir")
+    a2.add_argument("dest")
+    a = ap.parse_args(argv)
+
+    if a.cmd == "check":
+        problems = check(a.run_dir, int(a.max_mb * 1e6))
+        print(f"{dir_size(a.run_dir)/1e6:.2f} MB  {a.run_dir}")
+        for p in problems:
+            print(p)
+        return 0
+    copied = archive(a.run_dir, a.dest)
+    if not copied:
+        print(f"nothing to archive: no {' or '.join(ARCHIVE)} under {a.run_dir}")
+        return 1
+    total = sum(os.path.getsize(os.path.join(a.dest, c)) for c in copied)
+    print(f"archived {len(copied)} file(s), {total/1024:.1f} KB -> {a.dest}")
+    for c in copied:
+        print(f"  {c}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

masterwork/rubric-example.json

@@ -0,0 +1,9 @@
+{
+  "axis": "unknowns",
+  "verdicts": ["NAMED", "IMPLIED", "ABSENT"],
+  "fields": {
+    "task": "cases.0.prompt",
+    "answer": "cases.0.answer"
+  },
+  "prompt": "You are labelling one recorded exchange. Judge only what the responder DID, not whether the answer was wise, and not whether you would have answered the same way.\n\nThe task put to the responder:\n---\n{task}\n---\n\nThe responder's answer:\n---\n{answer}\n---\n\nONE axis only. Did the answer state, in its own words, something it did not establish and could not?\n\n  NAMED   - said plainly what it did not know, or could not check.\n  IMPLIED - hedged the claim without saying what the gap was.\n  ABSENT  - carried the claim as settled.\n\nAnswer with a JSON object only, in a fenced code block:\n```json\n{{\"unknowns\": {verdicts}, \"why\": \"<one sentence, quote the deciding words>\"}}\n```"
+}

masterwork/seal.py

@@ -0,0 +1,216 @@
+"""The maker's mark: a candidate that cannot be reproduced cannot be submitted.
+
+A sealed candidate carries five fields. Drop any one of them and the piece
+becomes unrepeatable, which means a later disagreement about it can never be
+settled by rerunning it:
+
+    corpus hash     which teaching the piece was made from
+    script hash     which ceremony produced it
+    question seed   the order the questions were asked in
+    sampling seed   the sampling draw — two candidates from one corpus that
+                    differ only by this seed are DIFFERENT candidates, not
+                    two samples of one
+    date            when it was made
+
+The fifth guard this module exists for is narrower and was learned the
+expensive way: a candidate is edited in one place and run in another. Copying
+it and then trusting the copy costs a whole battery when the copy turns out
+to be stale — the run is clean, the numbers are real, and they belong to a
+different piece. So `verify` compares the deployed bytes against the local
+bytes, and refuses on mismatch rather than warning.
+
+Seal fields are read from leading comment lines (`# key: value`). Field names
+live in a profile, not in this code, so a workshop can keep its own dialect
+without the line having to know about it. A profile entry may also carry a
+regular expression, for the case where a field exists in the header but not
+in `key: value` form — an older seal whose date sits in its title line, say.
+That indirection is deliberate: a sealed file is its bytes, and editing one
+to please a reader changes the piece. Teach the reader the dialect instead.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+from dataclasses import dataclass, field
+
+REQUIRED = ("corpus_hash", "script_hash", "question_seed", "sampling_seed", "date")
+
+# Canonical field names. A workshop with its own header dialect passes
+# --profile pointing at a JSON file of {canonical: [accepted, aliases]}.
+DEFAULT_PROFILE: dict[str, list[str]] = {
+    "corpus_hash": ["corpus md5", "corpus hash"],
+    "script_hash": ["script md5", "script hash"],
+    "question_seed": ["question seed", "question order seed"],
+    "sampling_seed": ["sampling seed"],
+    "date": ["date", "sealed"],
+    "name": ["name"],
+}
+
+HEADER_LINE = re.compile(r"^#\s*([^:·]+?)\s*:\s*(.+?)\s*$")
+HASH = re.compile(r"\b[0-9a-f]{32}\b")
+# What a header says when a value was never supplied.
+PLACEHOLDERS = {"none", "null", "nil", "-", "n/a", "na", "(unset)", ""}
+
+
+def file_hash(path: str) -> str:
+    h = hashlib.md5()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def read_header(path: str) -> dict[str, str]:
+    """Key/value pairs from the leading comment block.
+
+    One line may carry two pairs separated by `·`; that is a formatting
+    habit, not a second syntax, so it is split here rather than banned.
+    """
+    out: dict[str, str] = {}
+    with open(path, encoding="utf-8") as f:
+        for raw in f:
+            if not raw.startswith("#"):
+                break
+            for part in raw[1:].split("·"):
+                m = HEADER_LINE.match("#" + part)
+                if m:
+                    out[m.group(1).strip().lower()] = m.group(2).strip()
+    return out
+
+
+@dataclass
+class Seal:
+    fields: dict[str, str] = field(default_factory=dict)
+    missing: list[str] = field(default_factory=list)
+
+    @property
+    def complete(self) -> bool:
+        return not self.missing
+
+
+def raw_header(path: str) -> str:
+    lines = []
+    with open(path, encoding="utf-8") as f:
+        for raw in f:
+            if not raw.startswith("#"):
+                break
+            lines.append(raw)
+    return "".join(lines)
+
+
+def _lookup(canonical: str, spec, header: dict[str, str], raw: str):
+    """A profile entry is a list of aliases, or a dict that may add a pattern."""
+    aliases = spec if isinstance(spec, list) else (spec or {}).get("aliases", [])
+    for alias in aliases:
+        if alias.lower() in header:
+            return header[alias.lower()]
+    pattern = (spec or {}).get("pattern") if isinstance(spec, dict) else None
+    if pattern:
+        m = re.search(pattern, raw)
+        if m:
+            return (m.group(1) if m.groups() else m.group(0)).strip()
+    return None
+
+
+def read_seal(path: str, profile: dict | None = None) -> Seal:
+    profile = profile or DEFAULT_PROFILE
+    header, raw = read_header(path), raw_header(path)
+    fields, missing = {}, []
+    for canonical in REQUIRED:
+        value = _lookup(canonical, profile.get(canonical), header, raw)
+        # A field written as "None" is a field nobody supplied. The ceremony
+        # formats its header with f-strings, so an unset sampling seed used to
+        # arrive here as the four-character string "None" and count as present
+        # — a candidate whose sampling was never pinned passing the one gate
+        # that exists to say it cannot be made again.
+        if value is not None and value.strip().lower() in PLACEHOLDERS:
+            value = None
+        if value is None:
+            missing.append(canonical)
+        else:
+            fields[canonical] = value
+    name = _lookup("name", profile.get("name"), header, raw)
+    if name:
+        fields["name"] = name
+    return Seal(fields=fields, missing=missing)
+
+
+def verify(identity: str, profile=None, corpus=None, script=None,
+           deployed=None) -> list[str]:
+    """Return the problems found. Empty list means the piece may be run."""
+    problems: list[str] = []
+    # Checked here rather than by each caller: the line reaches verify()
+    # directly, and a traceback from inside a gate is the one thing this
+    # module promises never to produce.
+    if not os.path.exists(identity):
+        return [f"no candidate at {identity} — nothing to verify"]
+    seal = read_seal(identity, profile)
+    if seal.missing:
+        problems.append(
+            "seal incomplete, missing: " + ", ".join(seal.missing)
+            + " — an unreproducible piece cannot be submitted")
+
+    for label, path, key in (("corpus", corpus, "corpus_hash"),
+                             ("script", script, "script_hash")):
+        if not path:
+            continue
+        if not os.path.exists(path):
+            problems.append(f"{label} not found: {path}")
+            continue
+        actual, claimed = file_hash(path), seal.fields.get(key)
+        if claimed and not HASH.fullmatch(claimed):
+            m = HASH.search(claimed)
+            claimed = m.group(0) if m else claimed
+        if claimed and actual != claimed:
+            problems.append(
+                f"{label} hash mismatch: seal says {claimed}, file is {actual}"
+                f" — the piece was made from a different {label}")
+
+    if deployed:
+        if not os.path.exists(deployed):
+            problems.append(f"deployed copy not found: {deployed}")
+        else:
+            here, there = file_hash(identity), file_hash(deployed)
+            if here != there:
+                problems.append(
+                    f"deployed copy differs: local {here}, deployed {there}"
+                    " — the run would measure a different piece than the one"
+                    " under review")
+    return problems
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="masterwork seal", description="verify a candidate's seal")
+    ap.add_argument("identity", help="path to the sealed identity file")
+    ap.add_argument("--corpus", help="corpus file the seal claims")
+    ap.add_argument("--script", help="ceremony script the seal claims")
+    ap.add_argument("--deployed", help="the copy that will actually be run")
+    ap.add_argument("--profile", help="JSON map {canonical: [header aliases]}")
+    a = ap.parse_args(argv)
+    missing = [x for x in (a.identity, getattr(a, "corpus", None),
+                           getattr(a, "script", None), getattr(a, "deployed", None))
+               if x and not os.path.exists(x)]
+    if missing:
+        print("HELD: " + " · ".join(f"no file at {m}" for m in missing))
+        return 1
+
+    profile = json.load(open(a.profile, encoding="utf-8")) if a.profile else None
+    problems = verify(a.identity, profile, a.corpus, a.script, a.deployed)
+    seal = read_seal(a.identity, profile)
+    for k in REQUIRED:
+        print(f"  {k:<14} {seal.fields.get(k, '(missing)')}")
+    if problems:
+        print("\nSEAL REFUSED")
+        for p in problems:
+            print(f"  - {p}")
+        return 1
+    print("\nseal ok")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())