muster-memory 0.1.0__py3-none-any.whl

muster/__init__.py

@@ -0,0 +1,27 @@
+"""MUSTER: long-term memory for agents that keeps every word, sets its own strategy from the stored text, and makes no
+model call to build or grow the memory.
+
+    import muster
+    plan = muster.derive(records)        # how this store is cut, grouped and read, decided from its own text
+    print(muster.explain(plan))
+
+Heavy dependencies (numpy, torch, nltk, rank_bm25) are imported only when a step needs them.
+"""
+from ._version import __version__
+from .derive import derive, derive_collection, explain
+from .errors import (AssetMissing, EnvironmentRefused, MusterError, PlanError, PlanViolation, RecordError,
+                     StaleSelection, StoreLocked, StoreRefused)
+from .plan import ENCODER, RERANKER, ModelSpec, Plan
+from .records import DEFAULT_TEMPLATE, Record, numbered, serialize
+from . import presets
+from .memory import Memory, build, context, render, search
+from .models import Models
+from .render import Context, compose
+from .store import AddResult, Store
+from .cache import ScoreCache
+from .source_order import SourceOrder, OrderedRecords, normalize_ordered_records, build_declared
+
+__all__ = ["__version__", "build", "search", "render", "context", "compose", "Memory", "Models", "Context", "Store", "AddResult", "ScoreCache", "derive", "derive_collection", "explain", "Plan", "ModelSpec", "ENCODER", "RERANKER",
+           "Record", "DEFAULT_TEMPLATE", "numbered", "serialize", "presets", "MusterError", "PlanError", "RecordError",
+           "StoreRefused", "EnvironmentRefused", "AssetMissing", "StaleSelection", "PlanViolation", "StoreLocked",
+           "SourceOrder", "OrderedRecords", "normalize_ordered_records", "build_declared"]

muster/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

muster/adapters.py

@@ -0,0 +1,33 @@
+"""Turn common conversation shapes into records."""
+from .records import Record
+
+
+def chat(messages, *, session=None, start=0):
+    """[{"role": ..., "content": ..., "time"?: ...}] -> one Record per message, id "S<session>:<n>"."""
+    out = []
+    for n, m in enumerate(messages, start + 1):
+        out.append(Record(text=str(m.get("content", "")), id=("S%s:%d" % (session, n)) if session is not None else None,
+                          time=m.get("time") or m.get("timestamp"), speaker=m.get("role") or m.get("speaker"),
+                          session=None if session is None else str(session)))
+    return out
+
+
+def numbered_lines(lines, start=1):
+    """A numbered log: "1. <line>", "2. <line>", ... joined by newlines (one record per line)."""
+    return "\n".join("%d. %s" % (n, ln) for n, ln in enumerate(lines, start))
+
+
+def locomo(conversation):
+    """LoCoMo's own record line, one per turn: "<dia_id> [<session date_time>] <speaker>: <text>", plus
+    " [image: <caption>]" when the turn has an image. Sessions in numeric order."""
+    import re
+    keys = sorted((k for k in conversation if re.fullmatch(r"session_\d+", k)), key=lambda k: int(k.split("_")[1]))
+    out = []
+    for k in keys:
+        date = conversation[k + "_date_time"]
+        for t in conversation[k]:
+            text = t["text"]
+            if t.get("blip_caption"):
+                text += " [image: %s]" % t["blip_caption"]
+            out.append(Record(text=text, id=t["dia_id"], time=date, speaker=t["speaker"], session=k.split("_")[1]))
+    return out

muster/bpe.py

@@ -0,0 +1,130 @@
+"""Byte-pair encodings.
+
+cl100k_base counts tokens for the detection statistics; o200k_base cuts the store and counts unit sizes, as the
+paper's harness does (harness/lme_pf29.py:259, scripts/derive.py:134).
+
+The encodings come from tiktoken. If `muster fetch` has put the BPE files in muster's asset directory, they are read
+from there, each checked against the hash tiktoken itself pins, so a run never depends on tiktoken's download cache
+(which lives under the system temp directory). Otherwise tiktoken loads them in its usual way.
+"""
+import base64
+import hashlib
+import os
+
+from .errors import AssetMissing
+
+NAMES = ("cl100k_base", "o200k_base")
+_ENC = {}
+
+
+def asset_dir():
+    """Where `muster fetch` puts tokenizer and sentence-splitter files ($MUSTER_ASSETS, else a per-user data dir)."""
+    d = os.environ.get("MUSTER_ASSETS")
+    if d:
+        return d
+    base = os.environ.get("XDG_DATA_HOME") or os.path.join(os.path.expanduser("~"), ".local", "share")
+    return os.path.join(base, "muster")
+
+
+def _bpe_path(name):
+    return os.path.join(asset_dir(), "bpe", name + ".tiktoken")
+
+
+def _constructor(name, ranks_for):
+    """Run tiktoken's own constructor for `name` with its file loader replaced by `ranks_for(url, sha256)`.
+
+    The pattern string and the special tokens therefore come from the installed tiktoken, not from a copy."""
+    import tiktoken_ext.openai_public as op
+    if name not in op.ENCODING_CONSTRUCTORS:
+        raise AssetMissing("tiktoken has no encoding named %r" % name)
+    orig = op.load_tiktoken_bpe
+    op.load_tiktoken_bpe = lambda url, expected_hash=None: ranks_for(url, expected_hash)
+    try:
+        return op.ENCODING_CONSTRUCTORS[name]()
+    finally:
+        op.load_tiktoken_bpe = orig
+
+
+def source(name):
+    """(url, sha256) of the BPE file tiktoken's constructor names."""
+    seen = {}
+
+    def ranks_for(url, sha):
+        seen["url"], seen["sha256"] = url, sha
+        return {}
+    _constructor(name, ranks_for)
+    return seen["url"], seen["sha256"]
+
+
+def _from_file(name, path):
+    import tiktoken
+    data = open(path, "rb").read()
+
+    def ranks_for(url, sha):
+        if sha and hashlib.sha256(data).hexdigest() != sha:
+            raise AssetMissing("%s does not match the sha256 tiktoken pins (%s); run `muster fetch` again" % (path, sha))
+        return {base64.b64decode(tok): int(rank) for tok, rank in (ln.split() for ln in data.splitlines() if ln)}
+    return tiktoken.Encoding(**_constructor(name, ranks_for))
+
+
+def encoding(name):
+    """The tiktoken Encoding `name`, from muster's asset directory when it is there."""
+    enc = _ENC.get(name)
+    if enc is None:
+        p = _bpe_path(name)
+        if os.path.exists(p):
+            enc = _from_file(name, p)
+        else:
+            import tiktoken
+            enc = tiktoken.get_encoding(name)
+        _ENC[name] = enc
+    return enc
+
+
+def o200k():
+    return encoding("o200k_base")
+
+
+def cl100k():
+    return encoding("cl100k_base")
+
+
+# The three ways the harness encodes. They differ only on text that contains special-token strings.
+def count(enc, text):
+    """Token count with special-token strings read as plain text (lme_pf29.py:1127-1128, derive.py:72)."""
+    return len(enc.encode(text, disallowed_special=()))
+
+
+def encode_pack(enc, text):
+    """What the packer encodes with (lme_pf29.py:1273): only <|endoftext|> may appear; any other special string raises."""
+    return enc.encode(text, allowed_special={"<|endoftext|>"})
+
+
+def encode_label(enc, text):
+    """tiktoken's default mode, used for parent labels (lme_pf29.py:2237): any special-token string raises."""
+    return enc.encode(text)
+
+
+def fetch(names=NAMES, *, quiet=False):
+    """Download the BPE files into the asset directory and check each against tiktoken's pinned hash."""
+    import urllib.request
+    out = []
+    for name in names:
+        url, sha = source(name)
+        dest = _bpe_path(name)
+        if os.path.exists(dest) and hashlib.sha256(open(dest, "rb").read()).hexdigest() == sha:
+            out.append((name, dest, "present"))
+            continue
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        data = urllib.request.urlopen(url, timeout=60).read()
+        if hashlib.sha256(data).hexdigest() != sha:
+            raise AssetMissing("downloaded %s does not match its pinned sha256" % url)
+        tmp = dest + ".tmp"
+        with open(tmp, "wb") as f:
+            f.write(data)
+        os.replace(tmp, dest)
+        _ENC.pop(name, None)
+        out.append((name, dest, "fetched"))
+        if not quiet:
+            print("fetched %s -> %s" % (name, dest))
+    return out

muster/cache.py

@@ -0,0 +1,68 @@
+"""Text-keyed caches: reranker scores and embedding vectors, in one SQLite file or in memory.
+
+A score is keyed by (reranker, query, unit text) and a vector by (encoder, key text), so entries stay valid when a
+store grows and its unit indices move.
+"""
+import hashlib
+import sqlite3
+import threading
+
+
+def _sha(s):
+    return hashlib.sha1(s.encode("utf-8", "replace")).digest()
+
+
+def model_key(obj):
+    return "%s@%s" % (getattr(obj, "name", type(obj).__name__), getattr(obj, "revision", "") or "")
+
+
+class ScoreCache:
+    """Reranker scores. `path=None` keeps them in memory."""
+
+    def __init__(self, path=None, *, model="reranker"):
+        self.model = model
+        self._lock = threading.Lock()
+        self.db = sqlite3.connect(path or ":memory:", check_same_thread=False)
+        self.db.execute("CREATE TABLE IF NOT EXISTS scores(model TEXT, qsha BLOB, usha BLOB, score REAL, "
+                        "PRIMARY KEY (model, qsha, usha))")
+        self.db.commit()
+
+    def get_many(self, query, texts):
+        q = _sha(query)
+        out = {}
+        with self._lock:
+            for t in set(texts):
+                r = self.db.execute("SELECT score FROM scores WHERE model=? AND qsha=? AND usha=?",
+                                    (self.model, q, _sha(t))).fetchone()
+                if r is not None:
+                    out[t] = r[0]
+        return out
+
+    def put_many(self, query, scores):
+        q = _sha(query)
+        with self._lock:
+            self.db.executemany("INSERT OR REPLACE INTO scores VALUES (?,?,?,?)",
+                                [(self.model, q, _sha(t), float(v)) for t, v in scores.items()])
+            self.db.commit()
+
+
+class VectorCache:
+    """Embedding vectors (float32) keyed by key text."""
+
+    def __init__(self, db, model, *, create=True):
+        self.db, self.model = db, model
+        if create:
+            self.db.execute("CREATE TABLE IF NOT EXISTS vectors(model TEXT, ksha BLOB, dim INT, vec BLOB, "
+                            "PRIMARY KEY (model, ksha))")
+
+    def get_many(self, keys):
+        out = {}
+        for k in set(keys):
+            r = self.db.execute("SELECT vec FROM vectors WHERE model=? AND ksha=?", (self.model, _sha(k))).fetchone()
+            if r is not None:
+                out[k] = r[0]
+        return out
+
+    def put_many(self, items):
+        self.db.executemany("INSERT OR REPLACE INTO vectors VALUES (?,?,?,?)",
+                            [(self.model, _sha(k), len(v) // 4, v) for k, v in items.items()])

muster/cli.py

@@ -0,0 +1,248 @@
+"""The `muster` command.
+
+  muster derive PATH [--json]               the plan of a store (same output as scripts/derive.py, except a log
+                                            numbered under its own spelling, which the paper's script calls records)
+  muster query PATH "question"              build a memory from a file once and print what a reader gets
+  muster add --store S.muster FILE.jsonl    append records (one JSON object per line: text, speaker, time, ...)
+  muster context --store S.muster "query"   what a reader gets for a query
+  muster mcp --store S.muster               serve the store to MCP clients over stdio
+  muster http --store S.muster [--port N]   serve the store as a JSON API
+  muster verify cuts --data DIR             check the cut of every final-leg row against gate/cut_table.json
+  muster verify contexts --data DIR         rerun the paper's searches (exact mode) and compare every context hash
+  muster presets                            the paper's presets and the legs each stands for
+  muster fetch                              download the tokenizer and sentence-splitter files once, for offline use
+"""
+import argparse
+import json
+import sys
+
+from . import bpe
+from .derive import MIN_PARENTS, MIN_UNIT, NUMBERED_TAU, PROSE_STEP, RECORD_STEP, RECORD_TAU
+from .errors import MusterError
+from .io import read_store
+from .stats import LEAD_LINE_TOKENS, lead_serial, store_stats
+
+P = 4096
+
+
+def _decision(serial, record, total, lead=False):
+    """The dict scripts/derive.py prints, from the same three numbers (`lead`: numbered under its own spelling)."""
+    if serial >= NUMBERED_TAU or lead:
+        return dict(store="numbered log", unit_tokens=None, units=None, parent_tokens=None, parents=None,
+                    order="newest per slot wins", retrieval="mostly by slot",
+                    channel_floors=dict(meaning=2, keyword=0, parent=8, per_selected_units=10))
+    if record >= RECORD_TAU:
+        store, unit = "record store", max(P // RECORD_STEP, MIN_UNIT)
+    else:
+        store, unit = "prose", (P // PROSE_STEP if P > MIN_UNIT else P)
+    units = total // unit
+    mult = next((m for m in (8, 4, 2) if units // m >= MIN_PARENTS), None)
+    return dict(store=store, unit_tokens=unit, units=units, parent_tokens=unit * mult if mult else None,
+                parents=units // mult if mult else 0, order="none: all versions kept", retrieval="balanced, 3 channels",
+                channel_floors=dict(meaning=4, keyword=3, parent=3, per_selected_units=10))
+
+
+def _show(path, total, d):
+    print("MUSTER  %s  %s tokens" % (path, format(total, ",")))
+    if d["store"] == "numbered log":
+        rows = [("store", "numbered log"), ("units", "one per record"), ("parents", "one per slot"),
+                ("order", d["order"]), ("retrieval", d["retrieval"])]
+        if "leading_serial_share" in d:
+            rows.append(("serials", "each line's own prefix, shown as written"))
+    else:
+        if not d["units"]:
+            unit = "whole store (under one unit)"
+        elif d["store"] == "record store":
+            unit = "512 tokens, records whole"
+        else:
+            unit = "%s tokens, 4x as many" % format(d["unit_tokens"], ",")
+        par = "%s tokens, %d of them" % (format(d["parent_tokens"], ","), d["parents"]) if d["parent_tokens"] else "none (too short)"
+        rows = [("store", d["store"]), ("units", unit), ("parents", par), ("order", d["order"]),
+                ("retrieval", d["retrieval"])]
+    for k, v in rows:
+        print("  %-10s %s" % (k, v))
+
+
+def cmd_derive(a):
+    try:
+        text = read_store(a.path, a.column, a.row)
+    except FileNotFoundError:
+        sys.exit("no such file: %s" % a.path)
+    except UnicodeDecodeError:
+        sys.exit("%s is not UTF-8 text; save it as UTF-8 and run again" % a.path)
+    except MusterError as e:
+        sys.exit(str(e))
+    st = store_stats(text, P)
+    if st.tokens == 0:
+        sys.exit("%s is empty: nothing to configure" % a.path)
+    share, med = lead_serial(text) if st.serial < NUMBERED_TAU else (0.0, None)
+    lead = med is not None and med <= LEAD_LINE_TOKENS
+    d = _decision(st.serial, st.record_share, st.tokens, lead=lead)
+    if lead:
+        d = dict(leading_serial_share=round(share, 4), median_line_tokens=med, **d)
+    if a.json:
+        print(json.dumps(dict(path=a.path, tokens=st.tokens, serial_fraction=round(st.serial, 4),
+                              record_share=round(st.record_share, 4), **d), indent=1))
+    else:
+        _show(a.path, st.tokens, d)
+
+
+def cmd_verify(a):
+    import os
+    from .compat.verify import verify_cuts
+    root = a.repo or os.getcwd()
+    legs = set(a.legs.split(",")) if a.legs else None
+    if a.what == "cuts":
+        ok = verify_cuts(a.data, os.path.join(root, "gate", "cut_table.json"), legs=legs)
+    else:
+        from .compat.verify import verify_contexts
+        ok = verify_contexts(a.data, os.path.join(root, "gate", "ctx_table.jsonl"),
+                             os.path.join(root, "gate", "cut_table.json"), legs=legs,
+                             cache_dirs=[d for d in (a.cache or "").split(":") if d], nq=a.nq or None)
+    sys.exit(0 if ok else 1)
+
+
+def cmd_presets(a):
+    from . import presets
+    by = {}
+    for tag, leg in presets.legs().items():
+        by.setdefault(leg["preset"], []).append(tag)
+    for name in presets.names():
+        p = presets.get(name)
+        shape = ("%s/%s/%s" % (p.atom, p.chunk, p.mscale)) if p.atom else p.branch
+        print("%-11s %-9s %-16s U=%-3d %-13s %-8s %s" % (name, p.branch, shape, p.U, p.policy, p.render,
+                                                        ", ".join(sorted(by.get(name, [])))))
+
+
+def cmd_fetch(a):
+    for name, dest, what in bpe.fetch(quiet=True):
+        print("%-12s %-8s %s" % (name, what, dest))
+    from .cut import _ensure_punkt
+    _ensure_punkt(exact=False)
+    print("punkt_tab    ready")
+    if a.models:
+        from huggingface_hub import snapshot_download
+        from .plan import ENCODER, RERANKER
+        for spec in (ENCODER, RERANKER):
+            path = snapshot_download(spec.name, revision=spec.revision)
+            print("%-12s ready    %s" % (spec.name.split("/")[-1], path))
+
+
+def _models(a):
+    if getattr(a, "fake_models", False):
+        from .models import Models
+        return Models.fake(32)
+    return None
+
+
+def cmd_add(a):
+    from .store import Store
+    recs = [json.loads(x) for x in open(a.file, encoding="utf-8") if x.strip()] if a.file != "-" else \
+        [json.loads(x) for x in sys.stdin if x.strip()]
+    with Store.open(a.store, models=_models(a)) as st:
+        r = st.add(recs, request_id=a.request_id)
+    print(json.dumps({"version": r.version, "duplicate": r.duplicate, "n_records": r.n_records}))
+
+
+def cmd_context(a):
+    from .store import Store
+    with Store.open(a.store, models=_models(a), readonly=True) as st:
+        ctx = st.context(a.query, U=a.units or None)
+    print(ctx.text)
+
+
+def cmd_query(a):
+    from .memory import build
+    from .records import as_record
+    if a.declared_records:
+        from .source_order import build_declared
+        with open(a.path, encoding="utf-8") as source:
+            envelope = json.load(source)
+        mem = build_declared(envelope, models=_models(a))
+    elif a.path.endswith(".jsonl") and not a.column_given:
+        store = [as_record(json.loads(x)) for x in open(a.path, encoding="utf-8") if x.strip()]
+        mem = build(store, models=_models(a))
+    else:
+        store = read_store(a.path, a.column, a.row)
+        mem = build(store, models=_models(a))
+    if a.explain:
+        from .derive import explain
+        print(explain(mem.plan), file=sys.stderr)
+    print(mem.context(a.question, U=a.units or None).text)
+
+
+def cmd_mcp(a):
+    from .serve import Service, mcp_server
+    from .store import Store
+    mcp_server(Service(Store.open(a.store, models=_models(a)))).run(transport=a.transport)
+
+
+def cmd_http(a):
+    from .serve import Service, http_server
+    from .store import Store
+    srv = http_server(Service(Store.open(a.store, models=_models(a)),
+                              allow_declared_records=a.allow_declared_records), a.host, a.port)
+    print("muster http on %s:%d (POST /v1/add /v1/context /v1/search, GET /v1/info)" % (a.host, a.port), flush=True)
+    srv.serve_forever()
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="muster", description="MUSTER memory: derive, presets, fetch")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    d = sub.add_parser("derive", help="the plan of a store, from its own text")
+    d.add_argument("path")
+    d.add_argument("--column", default="context")
+    d.add_argument("--row", type=int, default=0)
+    d.add_argument("--json", action="store_true")
+    d.set_defaults(fn=cmd_derive)
+    q = sub.add_parser("query", help="build a memory from a file and print the context for a question")
+    q.add_argument("path")
+    q.add_argument("question")
+    q.add_argument("--units", type=int, default=0)
+    q.add_argument("--column", default=None)
+    q.add_argument("--row", type=int, default=0)
+    q.add_argument("--explain", action="store_true", help="print the plan and why, to stderr")
+    q.add_argument("--declared-records", action="store_true",
+                   help="native opt-in: JSON envelope with explicit record boundaries and source-order provenance")
+    q.add_argument("--fake-models", action="store_true", help=argparse.SUPPRESS)
+    q.set_defaults(fn=cmd_query)
+    for name, fn, extra in (("add", cmd_add, ("file",)), ("context", cmd_context, ("query",)),
+                            ("mcp", cmd_mcp, ()), ("http", cmd_http, ())):
+        p = sub.add_parser(name)
+        p.add_argument("--store", required=True)
+        p.add_argument("--fake-models", action="store_true", help=argparse.SUPPRESS)
+        for x in extra:
+            p.add_argument(x)
+        if name == "add":
+            p.add_argument("--request-id", default=None)
+        if name == "context":
+            p.add_argument("--units", type=int, default=0)
+        if name == "mcp":
+            p.add_argument("--transport", default="stdio", choices=["stdio", "streamable-http"])
+        if name == "http":
+            p.add_argument("--host", default="127.0.0.1")
+            p.add_argument("--port", type=int, default=8765)
+            p.add_argument("--allow-declared-records", action="store_true",
+                           help="enable stateless /v1/declared-context for explicit source-order envelopes")
+        p.set_defaults(fn=fn)
+    v = sub.add_parser("verify", help="check the library against the paper's tables (no benchmark text needed)")
+    v.add_argument("what", choices=["cuts", "contexts"])
+    v.add_argument("--cache", default="", help="contexts: colon-separated directories of published reranker caches")
+    v.add_argument("--nq", type=int, default=0, help="contexts: at most this many questions per store")
+    v.add_argument("--data", required=True, help="directory with the benchmark files at the legs' paths")
+    v.add_argument("--repo", default=None, help="repository root holding gate/ (default: the current directory)")
+    v.add_argument("--legs", default="", help="comma-separated leg tags (default: all)")
+    v.set_defaults(fn=cmd_verify)
+    sub.add_parser("presets", help="the paper's presets").set_defaults(fn=cmd_presets)
+    f = sub.add_parser("fetch", help="download tokenizer and sentence-splitter files (and, with --models, both models)")
+    f.add_argument("--models", action="store_true", help="also download bge-m3 and bge-reranker-v2-m3 at the pinned revisions")
+    f.set_defaults(fn=cmd_fetch)
+    a = ap.parse_args(argv)
+    if a.cmd == "query":
+        a.column_given = a.column is not None
+        a.column = a.column or "context"
+    a.fn(a)
+
+
+if __name__ == "__main__":
+    main()

muster/compat/__init__.py

@@ -0,0 +1 @@
+"""Reproduction tools: the paper's legs, its reranker-cache files, its bundles. Not needed to use MUSTER."""

muster/compat/bundles.py

@@ -0,0 +1,145 @@
+"""Bundles: one JSON line per question with the context a published read consumed.
+
+Library versions of harness/make_mab_bundle.py, harness/make_bundle.py, harness/render_rule.py (R4) and the
+FactConsolidation tag strip of gate/run_mab_ctx.sh. They write the same bytes, so a regenerated file can be compared
+with a published one by its sha256.
+"""
+import collections
+import glob
+import hashlib
+import json
+import os
+
+from .recover import index80, recover_ids
+from ..render import per_parent_groups
+
+
+def _h(s):
+    return hashlib.sha1(s.encode()).hexdigest()[:12]
+
+
+def _gold(a):
+    return [str(x) for x in a] if hasattr(a, "__len__") and not isinstance(a, str) else [str(a)]
+
+
+def read_dump(path, *, on_duplicate="error"):
+    """{(row, qi): record} of a DRY dump. A repeated (row, qi) is an error unless on_duplicate="last"."""
+    recs, dup = {}, 0
+    for line in open(path):
+        if not line.strip():
+            continue
+        r = json.loads(line)
+        k = (r["row"], r["qi"])
+        dup += k in recs
+        recs[k] = r
+    if dup and on_duplicate == "error":
+        raise ValueError("%s: %d duplicated (row, qi)" % (path, dup))
+    return recs, dup
+
+
+def bundle_mab(dump_path, *, tag, arm, df, nq, out, dsname, view="prune_annot"):
+    """make_mab_bundle.py: the scored arm's context per question, checked against its recorded hash."""
+    V = arm + "|" + view
+    recs, _ = read_dump(dump_path)
+    if len(recs) != nq:
+        raise ValueError("%s: partial export %d/%d" % (tag, len(recs), nq))
+    n = 0
+    with open(out, "w") as fo:
+        for k in sorted(recs):
+            r = recs[k]
+            row, qi = k
+            ctx = r["ctx|" + V]
+            h = _h(ctx)
+            if h != r["ctxhash|" + V]:
+                raise ValueError("%s %s: context does not hash to its recorded ctxhash" % (tag, k))
+            m = df["metadata"].iloc[row]
+            qs = list(df["questions"].iloc[row])
+            ans = df["answers"].iloc[row]
+            ids = list(m["question_ids"]) if m.get("question_ids") is not None else []
+            lines = [x for x in ctx.split("\n") if x.strip()]
+            c = ctx.count("[CURRENT]")
+            fo.write(json.dumps(dict(row=row, qi=qi, qt=r["qt"], dataset=dsname, question=str(qs[qi]), gold=_gold(ans[qi]),
+                                     gold_str=r["gold"], question_id=str(ids[qi]) if qi < len(ids) else "", arm=V,
+                                     src_tag=tag, ctxhash=h, tok=(r.get("tok_by_arm") or {}).get(V), n_lines=len(lines),
+                                     n_current=c, n_units=r.get("n_units"), context=ctx), ensure_ascii=False) + "\n")
+            n += 1
+    return n
+
+
+def bundle_beam(dump_paths, *, df, out, arm="flr_4_3_3|prune_annot"):
+    """make_bundle.py: every question of every conversation, in parquet order. Later shards win a repeated key,
+    as the original did; the repeats are returned so a caller can report them."""
+    recs, dups = {}, []
+    for f in dump_paths:
+        for line in open(f):
+            r = json.loads(line)
+            k = (r["row"], r["qi"])
+            if k in recs:
+                dups.append((k, recs[k]["ctxhash|" + arm] == r["ctxhash|" + arm]))
+            recs[k] = r
+    nq = sum(len(df.iloc[i]["questions"]) for i in range(len(df)))
+    if len(recs) != nq:
+        raise ValueError("partial export: %d/%d" % (len(recs), nq))
+    n = 0
+    with open(out, "w") as fo:
+        for row in range(len(df)):
+            m = dict(df.iloc[row]["metadata"])
+            qs = list(df.iloc[row]["questions"])
+            ans = df.iloc[row]["answers"]
+            for qi in range(len(qs)):
+                r = recs[(row, qi)]
+                ctx = r["ctx|" + arm]
+                if _h(ctx) != r["ctxhash|" + arm]:
+                    raise ValueError("%s: context does not hash to its recorded ctxhash" % ((row, qi),))
+                fo.write(json.dumps(dict(row=row, qi=qi, conv_id=str(m["conv_id"]), category=str(m["question_types"][qi]),
+                                         cat_index=int(m["cat_index"][qi]), question=str(qs[qi]), gold=_gold(ans[qi]),
+                                         rubric=[str(x) for x in m["rubric"][qi]], ctxhash=r["ctxhash|" + arm],
+                                         n_units=r.get("n_units"), tok=(r.get("tok_by_arm") or {}).get(arm),
+                                         context=ctx), ensure_ascii=False) + "\n")
+                n += 1
+    return n, dups
+
+
+def r4_bundle(bundle_in, out, units_of_row):
+    """render_rule.py R4: recover the ids each context holds, render one block per parent. `units_of_row(row)` returns
+    (units, parent) of that row's store under the leg's cut."""
+    stat = collections.Counter()
+    cache = {}
+    with open(out, "w") as fo:
+        for line in open(bundle_in):
+            r = json.loads(line)
+            row = r["row"]
+            if row not in cache:
+                u, p = units_of_row(row)
+                cache[row] = (u, p, index80(u))
+            units, parent, idx = cache[row]
+            ids = recover_ids(r["context"], units, parent, idx)
+            groups = per_parent_groups(ids, parent)
+            new = "\n".join("Memory %d:\n%s" % (n + 1, "\n".join(units[i] for i in b)) for n, b in enumerate(groups))
+            stat["same_as_input"] += new == r["context"]
+            r["context"] = new
+            r["ctxhash"] = _h(new)
+            fo.write(json.dumps(r, ensure_ascii=False) + "\n")
+            stat["n"] += 1
+    return dict(stat)
+
+
+def notag(bundle_in, out):
+    """FactConsolidation: drop the "[CURRENT] " tag the structural render writes before the newest record of a slot."""
+    rows = [json.loads(line) for line in open(bundle_in)]
+    n_tag = 0
+    for d in rows:
+        n_tag += d["context"].count("[CURRENT] ")
+        d["context"] = d["context"].replace("[CURRENT] ", "")
+        if "[CURRENT]" in d["context"]:
+            raise ValueError("a [CURRENT] tag without its trailing space survived the strip")
+        d["ctxhash"] = _h(d["context"])
+    with open(out, "w") as fo:
+        for d in rows:
+            fo.write(json.dumps(d, ensure_ascii=False) + "\n")
+    return len(rows), n_tag
+
+
+def shards(results_dir, tag):
+    return sorted(glob.glob(os.path.join(results_dir, "%s_s*.jsonl" % tag)) or
+                  glob.glob(os.path.join(results_dir, "%s_r*.jsonl" % tag)))