muster-memory 0.1.0__py3-none-any.whl

muster/cut.py

@@ -0,0 +1,291 @@
+"""Cutting: store text -> units, the parent of each unit, and its order.
+
+Three shapes, chosen by the plan:
+  record / prose  atoms (blank-line records, or sentences) packed greedily up to `chunk` o200k tokens; an atom is
+                  split only if it alone exceeds `chunk`. Parents label each unit with the `mscale`-token block its
+                  first token falls in ("m<k>"), or one parent per unit ("u<i>") when there is no parent layer.
+  numbered        one unit per "<serial>. <body>" line, serial stripped; order = the serial; parent = the slot the
+                  record updates ("s:" + entity | relation skeleton). A plan with serial "lead" reads each line's
+                  own prefix instead ("Step 3: <body>") and keeps it for the render.
+  rounds          a LongMemEval-style literal [date, turns, date, turns, ...]: one unit per user turn with the
+                  assistant reply, parent = the session.
+
+In exact mode every step is the harness's (harness/lme_pf29.py:1224-1291, 2089-2146, 2232-2247, 1059-1101), and the
+harness's refusals become StoreRefused. Native mode departs only where the harness would crash or split a character.
+"""
+import ast
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Optional, Tuple
+
+from . import bpe
+from .errors import AssetMissing, PlanError, StoreRefused
+from .stats import LEAD, lead_match
+
+_BLANK = re.compile(r"\n\s*\n")
+JOIN = {"blank": "\n\n", "newline": "\n", "sentence": " "}
+
+
+@dataclass(frozen=True)
+class Cut:
+    units: Tuple[str, ...]
+    keys: Tuple[str, ...]                 # what is embedded: the unit, except rounds (the user turn)
+    parent: tuple                         # "m%d", "u%d", "s:<slot>" or an int session, as the harness labels them
+    order: tuple                          # the serial on numbered logs, else 0..N-1
+    spans: Tuple[Optional[Tuple[int, int]], ...]
+    n_o200k: Tuple[int, ...]
+    sep: Optional[str]
+    diagnostics: dict = field(default_factory=dict)
+    prefixes: Optional[Tuple[str, ...]] = None   # serial "lead": each line's own prefix, "" where it has none
+
+    def __len__(self):
+        return len(self.units)
+
+
+# ---------------------------------------------------------------- atoms
+def _ensure_punkt(exact):
+    import nltk
+    d = os.path.join(bpe.asset_dir(), "nltk_data")
+    if os.path.isdir(d) and d not in nltk.data.path:
+        nltk.data.path.insert(0, d)
+    try:
+        nltk.data.find("tokenizers/punkt_tab")
+    except LookupError:
+        if exact:
+            raise AssetMissing("sentence splitting needs nltk's punkt_tab; run `muster fetch`")
+        os.makedirs(d, exist_ok=True)
+        nltk.download("punkt_tab", download_dir=d, quiet=True)
+        if d not in nltk.data.path:
+            nltk.data.path.insert(0, d)
+    return nltk
+
+
+def _with_spans(text, pieces):
+    out, cur = [], 0
+    for p in pieces:
+        j = text.find(p, cur)
+        if j < 0:
+            out.append((p, None))
+        else:
+            out.append((p, (j, j + len(p))))
+            cur = j + len(p)
+    return out
+
+
+def atoms(text: str, kind: str, *, exact: bool = False):
+    """[(atom, (start, end) or None)] for kind blank, newline or sentence (lme_pf29.py:1224-1236)."""
+    if kind == "blank":
+        out, pos = [], 0
+        for m in _BLANK.finditer(text):
+            piece = text[pos:m.start()]
+            if piece.strip():
+                out.append((piece, (pos, m.start())))
+            pos = m.end()
+        piece = text[pos:]
+        if piece.strip():
+            out.append((piece, (pos, len(text))))
+        return out
+    if kind == "newline":
+        out, pos = [], 0
+        for piece in text.split("\n"):
+            if piece.strip():
+                out.append((piece, (pos, pos + len(piece))))
+            pos += len(piece) + 1
+        return out
+    if kind == "sentence":
+        nltk = _ensure_punkt(exact)
+        return _with_spans(text, nltk.sent_tokenize(text))
+    raise PlanError("unknown atom kind %r" % kind)
+
+
+# ---------------------------------------------------------------- packing
+def _special_strings(enc, text):
+    return sorted(s for s in enc.special_tokens_set if s in text)
+
+
+def pack(atom_list, chunk: int, joiner: str, *, exact: bool = False):
+    """Greedy packing (lme_pf29.py:1255-1291). Returns [(unit, [atom indices])] and the oversize-atom count."""
+    enc = bpe.o200k()
+    out, cur, curi, n, over = [], [], [], 0, 0
+    for i, a in enumerate(atom_list):
+        if exact:
+            t = bpe.encode_pack(enc, a)
+        else:
+            t = enc.encode(a, disallowed_special=())
+        if len(t) > chunk:
+            over += 1
+            if cur:
+                out.append((joiner.join(cur), curi))
+                cur, curi, n = [], [], 0
+            if exact:
+                for o in range(0, len(t), chunk):
+                    out.append((enc.decode(t[o:o + chunk]), [i]))
+            else:                           # N5: slice on character boundaries, so the slices rebuild the atom
+                _, offs = enc.decode_with_offsets(t)
+                cuts = [offs[o] for o in range(0, len(t), chunk)] + [len(a)]
+                for s, e in zip(cuts, cuts[1:]):
+                    if e > s:
+                        out.append((a[s:e], [i]))
+            continue
+        if n + len(t) > chunk and cur:
+            out.append((joiner.join(cur), curi))
+            cur, curi, n = [], [], 0
+        cur.append(a)
+        curi.append(i)
+        n += len(t)
+    if cur:
+        out.append((joiner.join(cur), curi))
+    return out, over
+
+
+# ---------------------------------------------------------------- numbered logs (AUTOSTRUCT, AS_SLOT=5)
+_SERIAL = re.compile(r"^\s*(\d+)\.\s+(.+)$", re.DOTALL)
+_CAPRE = re.compile(r"\b([A-Z][\w.'-]+(?:\s+(?:of|the|and|de|van|von|di)?\s*[A-Z][\w.'-]+)*)\b")
+_CAPSTOP = {"The", "A", "An", "In", "On", "At", "Of", "This", "That", "These", "Those", "It", "Its", "Which",
+            "What", "Who", "Where", "When", "There", "He", "She", "They", "His", "Her", "Their"}
+AS_MINREC = 0.8
+
+
+def entities(t):
+    """Capitalised spans, minus sentence-initial function words (lme_pf29.py:2107-2110)."""
+    return [m.group(1) for m in _CAPRE.finditer(t or "") if m.group(1).split()[0] not in _CAPSTOP]
+
+
+def slot_of(body):
+    """The slot a numbered record updates: its first entity, then the sentence with every entity masked."""
+    e = entities(body)
+    sub = re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", e[0].lower())).strip() if e else ""
+    sk = re.sub(r"\s+", " ", _CAPRE.sub("_", body or "")).strip().rstrip(".").strip()
+    return sub + "|" + sk
+
+
+def _numbered(text, exact, serial="dot"):
+    lines = [ln for ln in text.split("\n") if ln.strip()]
+    lead = serial == "lead"
+    ms = [(lead_match(ln) if not exact else LEAD.match(ln)) if lead else _SERIAL.match(ln) for ln in lines]
+    share = sum(1 for x in ms if x) / float(max(len(lines), 1))
+    if share < AS_MINREC:
+        if exact:
+            raise StoreRefused("numbered log: only %.3f of lines are `<serial>. <body>` (needs %.1f; lme_pf29.py:2126)"
+                               % (share, AS_MINREC))
+        raise PlanError("not a numbered log: %.3f of all lines carry a %sserial" % (share, "leading " if lead else ""))
+    units, order, parent, prefixes = [], [], [], []
+    unmarked = []
+    for x, ln in zip(ms, lines):
+        if x and lead:
+            ser, body, pre = int(x.group(2)), ln[x.end():].strip(), ln[:x.end()]
+        elif x:
+            ser, body, pre = int(x.group(1)), x.group(2).strip(), None
+        else:
+            ser, body, pre = len(units), ln.strip(), ""
+            if lead and not exact:
+                ser, body = -1, ln  # preserve the literal line; no invented revision clock
+                unmarked.append(len(units))
+        units.append(body)
+        order.append(ser)
+        parent.append("u:rawline%d" % (len(units) - 1) if lead and not exact and x is None else "s:" + slot_of(body))
+        prefixes.append(pre)
+    spans = tuple(sp for _, sp in _with_spans(text, lines))
+    enc = bpe.o200k()
+    serials = [o for o, x in zip(order, ms) if x]
+    diag = dict(numbered_share=round(share, 4), distinct_serial_share=round(len(set(serials)) / float(max(len(serials), 1)), 4),
+                n_parents=len(set(parent)))
+    if unmarked:
+        diag["unmarked_records"] = unmarked
+    return Cut(tuple(units), tuple(units), tuple(parent), tuple(order), spans,
+               tuple(bpe.count(enc, u) for u in units), "\n", diag, tuple(prefixes) if lead else None)
+
+
+# ---------------------------------------------------------------- rounds (LongMemEval literal)
+def sessions_from_literal(literal: str):
+    """[(date, turns)] from "[date, turns, date, turns, ...]" (lme_pf29.py:1059-1072)."""
+    lit = ast.literal_eval(literal)
+    out, i = [], 0
+    while i < len(lit):
+        if isinstance(lit[i], str) and i + 1 < len(lit) and isinstance(lit[i + 1], list):
+            out.append((lit[i], lit[i + 1]))
+            i += 2
+        else:
+            i += 1
+    return out
+
+
+def rounds_of(sessions):
+    """One unit per user turn with the assistant turn that answers it (lme_pf29.py:1075-1101)."""
+    rtext, rkey, sess_id, rdate = [], [], [], []
+    for si, (date, turns) in enumerate(sessions):
+        for ti, t in enumerate(turns):
+            if not isinstance(t, dict) or t.get("role") != "user":
+                continue
+            nxt = ""
+            if ti + 1 < len(turns) and isinstance(turns[ti + 1], dict) and turns[ti + 1].get("role") != "user":
+                nxt = turns[ti + 1].get("content", "")
+            rtext.append(("[" + date + "] " + t.get("content", "") + "\n" + nxt).strip())
+            rkey.append(t.get("content", "").strip())
+            sess_id.append(si)
+            rdate.append(date)
+    return rtext, rkey, sess_id, rdate
+
+
+def _rounds(literal):
+    sess = sessions_from_literal(literal)
+    rtext, rkey, sid, rdate = rounds_of(sess)
+    enc = bpe.o200k()
+    return Cut(tuple(rtext), tuple(rkey), tuple(sid), tuple(range(len(rtext))), tuple([None] * len(rtext)),
+               tuple(bpe.count(enc, u) for u in rtext), None, dict(n_sessions=len(sess), dates=tuple(rdate)))
+
+
+# ---------------------------------------------------------------- entry point
+def cut(text: str, plan, *, exact: Optional[bool] = None) -> Cut:
+    """Cut `text` under `plan`. `exact` defaults to plan.mode == "exact"."""
+    exact = (plan.mode == "exact") if exact is None else exact
+    if plan.branch == "empty":
+        return Cut((), (), (), (), (), (), None, {})
+    if plan.branch == "numbered":
+        return _numbered(text, exact, plan.serial)
+    if plan.branch == "rounds":
+        return _rounds(text)
+    enc = bpe.o200k()
+    if exact:
+        bad = [s for s in _special_strings(enc, text) if s != "<|endoftext|>"]
+        if bad:
+            raise StoreRefused("the store contains special-token text %r, which the harness's packer cannot encode "
+                               "(lme_pf29.py:1273)" % bad[0])
+    at = atoms(text, plan.atom, exact=exact)
+    packed, over = pack([a for a, _ in at], plan.chunk, JOIN[plan.atom], exact=exact)
+    units = [u for u, _ in packed]
+    spans = []
+    for _, idx in packed:
+        s, e = at[idx[0]][1], at[idx[-1]][1]
+        spans.append((s[0], e[1]) if (s and e) else None)
+    diag = dict(atoms=len(at), oversize_atoms=over)
+    if plan.mscale:
+        if exact and not units:
+            raise StoreRefused("MSCALE produced no units (lme_pf29.py:2239)")
+        parent, cum = [], 0
+        for u in units:
+            parent.append("m%d" % (cum // plan.mscale))
+            if exact:
+                try:
+                    cum += len(bpe.encode_label(enc, u))
+                except ValueError as e:
+                    raise StoreRefused("a unit holds special-token text the harness's parent labels cannot encode "
+                                       "(lme_pf29.py:2237): %s" % e)
+            else:
+                cum += bpe.count(enc, u)
+        if len(set(parent)) >= len(units):
+            if exact:
+                raise StoreRefused("MSCALE produced a flat layer (%d units, %d parents) (lme_pf29.py:2240-2243)"
+                                   % (len(units), len(set(parent))))
+            parent = ["u%d" % i for i in range(len(units))]
+            diag["flat_parents"] = True
+        if exact:
+            ns = lambda t: len(re.sub(r"\s+", "", t))
+            lost = ns(text) - ns("".join(units))
+            if abs(lost) > 0.01 * max(1, ns(text)):
+                raise StoreRefused("MSCALE lost or duplicated text: %d non-space characters (lme_pf29.py:2244-2247)" % lost)
+    else:
+        parent = ["u%d" % i for i in range(len(units))]
+    return Cut(tuple(units), tuple(units), tuple(parent), tuple(range(len(units))), tuple(spans),
+               tuple(bpe.count(enc, u) for u in units), "\n\n" if plan.atom != "newline" else "\n", diag)

muster/derive.py

@@ -0,0 +1,146 @@
+"""derive: a store's plan from its own text, with no model, no query and nothing to tune.
+
+The rule (paper, Algorithm 1, Part I; thresholds fixed before any experiment, harness/derive_config.py):
+
+  numbered log  >= 80% of lines open with a serial "12. ..."  -> one unit per record, parent = the slot it updates,
+                the newest serial of a slot wins
+  record store  >= 90% of the text sits in blank-line records of at most the reference chunk P
+                -> units of max(P/8, 512) tokens with records kept whole
+  prose         otherwise -> units of P/4 tokens (step 8: the P-token unit searched at a quarter of its size with four
+                times as many units, so the token budget is unchanged)
+
+  Parents are 8 units, or 4 or 2 if fewer than ten parents would remain; else there is no parent layer.
+
+Added after the paper (2026-11, thresholds set on public benchmark logs; `derive` only, not `derive_collection` or the
+presets): a store the serial fraction misses is still a numbered log when >= 95% of its lines after the first carry a
+leading serial one above the previous line's, written in the log's own style ("Step 3: ..."), and its median line is
+at most 20 tokens. The serial is read from that prefix, and the reader sees each line as written.
+"""
+from typing import Iterable, Sequence, Union
+from dataclasses import replace
+
+from .errors import PlanError
+from .plan import Plan
+from .records import DEFAULT_TEMPLATE, as_record, normalize_newlines, serialize
+from .record_serial import body_serial_stats
+from .stats import LEAD_LINE_TOKENS, LEAD_TAU, Stats, collection_stats, lead_serial, store_stats
+
+NUMBERED_TAU, RECORD_TAU, MIN_PARENTS = 0.80, 0.90, 10
+RECORD_STEP, MIN_UNIT, PROSE_STEP = 8, 512, 4
+U_DEFAULT = {"numbered": 10, "record": 10, "prose": 40}
+
+
+def store_text(store, *, template=DEFAULT_TEMPLATE, normalize=True):
+    """(text, record spans or None, records or None) for a string or an iterable of records."""
+    if isinstance(store, str):
+        return (normalize_newlines(store) if normalize else store), None, None
+    recs = [as_record(x) for x in store]
+    text, spans = serialize(recs, template=template, normalize=normalize)
+    return text, spans, recs
+
+
+def _parents(units_est, chunk):
+    for m in (8, 4, 2):
+        if units_est // m >= MIN_PARENTS:
+            return chunk * m, units_est // m, m
+    return 0, 0, None
+
+
+def _plan(branch, chunk, units_est, st: Stats, P, trace, serial="dot"):
+    if branch == "numbered":
+        if serial == "dot":
+            trace.append("serial fraction %.4f >= %.2f: numbered log; one unit per record, parent = slot, newest "
+                         "serial wins" % (st.serial, NUMBERED_TAU))
+        return Plan(branch="numbered", atom=None, chunk=None, mscale=0, U=U_DEFAULT["numbered"],
+                    policy="attprio_sess", render="numbered", supersede="numbered", reference_chunk=P,
+                    stats=st.to_dict(), trace=tuple(trace), serial=serial)
+    atom = "blank" if branch == "record" else "sentence"
+    ms, npar, m = _parents(units_est, chunk)
+    trace.append("%d units estimated at %d tokens -> %s" % (
+        units_est, chunk, ("parents of %d units (%d tokens), %d of them" % (m, ms, npar)) if ms
+        else "no parent layer (fewer than %d parents at 2 units each)" % MIN_PARENTS))
+    return Plan(branch=branch, atom=atom, chunk=chunk, mscale=ms, U=U_DEFAULT[branch], policy="flrs", render="r4",
+                reference_chunk=P, stats=st.to_dict(), trace=tuple(trace))
+
+
+def derive(store, *, reference_chunk: int = 4096, template: str = DEFAULT_TEMPLATE,
+           normalize_newlines: bool = True) -> Plan:
+    """The plan of one store, with the documented native serial-format extension."""
+    P = int(reference_chunk)
+    text, _, recs = store_text(store, template=template, normalize=normalize_newlines)
+    st = store_stats(text, P)
+    body_stats = body_serial_stats(recs, normalize=normalize_newlines) if recs is not None else None
+    if body_stats is not None:
+        st = replace(st, serial=body_stats[0])  # metadata never supplies a record serial
+    trace = ["serial fraction %.4f, record share %.4f, %d tokens (cl100k_base)" % (st.serial, st.record_share, st.tokens)]
+    if st.tokens == 0:
+        trace.append("empty store")
+        return Plan(branch="empty", atom=None, chunk=None, mscale=0, U=0, policy="flr", render="flat",
+                    reference_chunk=P, stats=st.to_dict(), trace=tuple(trace))
+    if st.serial >= NUMBERED_TAU:
+        if body_stats is not None:
+            trace.append("Serials are read only at the start of original record bodies; metadata is retrieval/display data.")
+        return _plan("numbered", None, 0, st, P, trace)
+    share, med = lead_serial(text) if body_stats is None else body_stats[1:]
+    if med is not None and med <= LEAD_LINE_TOKENS:
+        if body_stats is None:
+            trace.append("leading serial one above the previous line's on %.4f >= %.2f of lines, median line %g <= %d "
+                         "tokens: numbered log under its own spelling; one unit per record, parent = slot, newest serial "
+                         "wins" % (share, LEAD_TAU, med, LEAD_LINE_TOKENS))
+        else:
+            trace.append("leading body serial one above the previous record's on %.4f >= %.2f of original record "
+                         "pairs, median entire record body %g <= %d tokens: numbered records; metadata kept for "
+                         "retrieval/display, not slot/order" % (share, LEAD_TAU, med, LEAD_LINE_TOKENS))
+        return _plan("numbered", None, 0, st, P, trace, serial="lead")
+    if st.record_share >= RECORD_TAU:
+        chunk = max(P // RECORD_STEP, MIN_UNIT)
+        trace.append("record share %.4f >= %.2f: record store; units of %d tokens, records kept whole"
+                     % (st.record_share, RECORD_TAU, chunk))
+        return _plan("record", chunk, st.tokens // chunk, st, P, trace)
+    chunk = P // PROSE_STEP if P > MIN_UNIT else P
+    trace.append("record share %.4f < %.2f: prose; units of %d tokens (P/%d, step 8)"
+                 % (st.record_share, RECORD_TAU, chunk, PROSE_STEP))
+    return _plan("prose", chunk, st.tokens // chunk, st, P, trace)
+
+
+def derive_collection(stores: Sequence[str], *, reference_chunk: int = 4096, sample: int = 3, step8: bool = True,
+                      mode: str = "native") -> Plan:
+    """One plan for a benchmark cell from its first `sample` stores (harness/derive_config.py, then step 8)."""
+    P = int(reference_chunk)
+    if P not in (512, 4096):
+        raise PlanError("reference_chunk must be 512 or 4096, MemoryAgentBench's protocol sizes; got %d" % P)
+    st = collection_stats(stores, P, sample=sample, mode=mode)
+    trace = ["%d stores: serial fraction %.4f (mean), record share %.4f (pooled), %d tokens per store"
+             % (st.rows, st.serial, st.record_share, st.tokens)]
+    if st.serial >= NUMBERED_TAU:
+        return _plan("numbered", None, 0, st, P, trace)
+    record = st.record_share >= RECORD_TAU
+    chunk = P // RECORD_STEP if (record and P // RECORD_STEP >= MIN_UNIT) else P
+    if not record and step8 and P > MIN_UNIT:
+        chunk = P // PROSE_STEP
+    trace.append("%s; units of %d tokens" % ("record store" if record else "prose", chunk))
+    return _plan("record" if record else "prose", chunk, max(1, st.tokens // chunk), st, P, trace)
+
+
+def explain(plan: Plan) -> str:
+    """A plain description of a plan, then the decision step by step."""
+    rows = [("store", {"numbered": "numbered log", "record": "record store", "prose": "prose", "rounds": "conversation rounds",
+                       "empty": "empty"}[plan.branch])]
+    if plan.source == "declared-order:record-boundaries-v1":
+        rows += [("units", "one per declared record"), ("parents", "one per source occurrence"),
+                 ("order", "declared source sequence; all versions kept")]
+    elif plan.branch == "numbered":
+        rows += [("units", "one per record"), ("parents", "one per slot"), ("order", "newest per slot wins")]
+        if plan.serial == "lead":
+            rows.append(("serials", "each line's own prefix, shown as written"))
+    elif plan.branch in ("record", "prose"):
+        rows += [("units", "%s tokens%s" % (format(plan.chunk, ","), ", records whole" if plan.atom == "blank" else "")),
+                 ("parents", ("%s tokens" % format(plan.mscale, ",")) if plan.mscale else "none")]
+    elif plan.branch == "rounds":
+        rows += [("units", "one user turn with its answer"), ("parents", "one per session")]
+    rows += [("read", "%d units per query" % plan.U), ("mode", plan.mode), ("source", plan.source)]
+    out = ["  %-10s %s" % r for r in rows]
+    if plan.trace:
+        out.append("  how:")
+        out += ["    - " + t for t in plan.trace]
+    return "\n".join(out)

muster/errors.py

@@ -0,0 +1,39 @@
+"""Every error the library raises is a MusterError, so a caller can catch one class."""
+
+
+class MusterError(Exception):
+    """Base class."""
+
+
+class PlanError(MusterError, ValueError):
+    """A plan that cannot be built or does not fit the store it was given."""
+
+
+class RecordError(MusterError, ValueError):
+    """A record the library cannot read (missing text, ambiguous time, multi-line numbered text)."""
+
+
+class StoreRefused(MusterError):
+    """Exact mode refuses a store or a step that the paper's harness refuses or would crash on.
+
+    The message names the harness line, so the refusal can be checked against the code that produced the paper."""
+
+
+class EnvironmentRefused(MusterError):
+    """Exact mode needs the gate machine's environment (hash seed off, pinned versions, pinned models)."""
+
+
+class AssetMissing(MusterError):
+    """A tokenizer or sentence-splitter file is missing or does not match its pinned hash. Run `muster fetch`."""
+
+
+class StaleSelection(MusterError):
+    """A selection made on one version of a memory was rendered on another."""
+
+
+class PlanViolation(MusterError):
+    """An add would break a frozen exact plan; nothing was changed."""
+
+
+class StoreLocked(MusterError):
+    """Another process holds the store's writer lock."""

muster/index.py

@@ -0,0 +1,102 @@
+"""The model-free index of a store: entities, the entity graph, IDF, slots, positions and BM25.
+
+Every structure is built with the harness's own expressions (src/real_e1_conflict_killer.py:26-54,
+harness/lme_pf29.py:2466-2525, 2592-2596; core2.py:40-61, 429-438). In exact mode the sets are kept as sets,
+because their iteration order under the paper's hash seed is part of the published output; native mode reads them
+in a canonical order wherever order can matter (muster/search.py).
+"""
+import collections
+import math
+import re
+
+_CAP = re.compile(r"\b([A-Z][\w.'-]+(?:\s+(?:of|the|and|de|van|von|di)?\s*[A-Z][\w.'-]+)*)\b")
+HUB_FRAC = 0.05
+ORDER_DISTINCT = 0.99
+
+
+def entities(text):
+    """Capitalised spans of more than two characters, as a set."""
+    return {m.strip() for m in _CAP.findall(text or "") if len(m.strip()) > 2}
+
+
+def build_entity_index(fact_ents, n_facts, hub_frac=HUB_FRAC):
+    """({entity: set of units}, hubs). An entity in more than max(3, 5% of units) units is a hub and leaves the index."""
+    ent_facts = {}
+    for i, es in enumerate(fact_ents):
+        for e in es:
+            ent_facts.setdefault(e, set()).add(i)
+    hub_cut = max(3, int(hub_frac * n_facts))
+    hubs = {e for e, fs in ent_facts.items() if len(fs) > hub_cut}
+    return {e: fs for e, fs in ent_facts.items() if e not in hubs}, hubs
+
+
+def slotkey(a):
+    return " ".join(re.sub(r"[^a-z0-9 ]", " ", str(a).lower()).split())
+
+
+def btok(t):
+    """BM25 tokens: lower case, punctuation to spaces, split on white space (BM25LC=1)."""
+    t = t.lower()
+    t = "".join(c if (c.isalnum() or c.isspace()) else " " for c in t)
+    return t.split()
+
+
+def newest_per_parent(pool, parent_of, order_of):
+    newest = {}
+    for i in pool:
+        k = parent_of[i]
+        if k is not None and (k not in newest or order_of[i] > order_of[newest[k]]):
+            newest[k] = i
+    return newest
+
+
+class Index:
+    """Built once per store version; a pure function of its units, keys, parents and order."""
+
+    def __init__(self, units, keys, parent, order, *, has_order=False):
+        N = self.N = len(units)
+        self.units, self.keys, self.parent = list(units), list(keys), list(parent)
+        self.r_ents = [entities(t) for t in self.keys]
+        self.ef, self.hubs = build_entity_index(self.r_ents, N)
+        self.idf = {e: math.log(N / max(len(v), 1)) for e, v in self.ef.items()}
+        self.r_slot = [slotkey(t) for t in self.keys]
+        sm = collections.defaultdict(list)
+        for i, s in enumerate(self.r_slot):
+            if s:
+                sm[s].append(i)
+        self.newest_of_slot = {k: max(v) for k, v in sm.items()}
+        # the order a structural resolver may trust: declared (numbered logs), and separating almost every unit
+        self.order = list(order) if has_order else list(range(N))
+        order_ok = has_order and len(set(self.order)) >= ORDER_DISTINCT * N
+        self.structural = bool(order_ok and len(set(self.parent)) < N)
+        self.children_of = {}
+        for i, p in enumerate(self.parent):
+            if p is not None:
+                self.children_of.setdefault(p, []).append(i)
+        self.pos_of, self.at = {}, {}
+        cnt = collections.Counter()
+        for i, si in enumerate(self.parent):
+            self.pos_of[i] = cnt[si]
+            self.at[(si, cnt[si])] = i
+            cnt[si] += 1
+        self.newest_row = newest_per_parent(range(N), self.parent, self.order) if self.structural else {}
+        self.adj_slot = {}
+        if self.structural:
+            ent_slots = collections.defaultdict(set)
+            for s2, i2 in self.newest_row.items():
+                for e2 in (self.r_ents[i2] - self.hubs):
+                    ent_slots[e2].add(s2)
+            cut = max(3, int(HUB_FRAC * max(len(self.newest_row), 1)))
+            ent_slots = {e2: v2 for e2, v2 in ent_slots.items() if len(v2) <= cut}
+            self.adj_slot = collections.defaultdict(set)
+            for e2, ss2 in ent_slots.items():
+                for s2 in ss2:
+                    self.adj_slot[s2] |= (ss2 - {s2})
+        self._bm25 = None
+
+    @property
+    def bm25(self):
+        if self._bm25 is None:
+            from rank_bm25 import BM25Okapi
+            self._bm25 = BM25Okapi([btok(t) for t in self.units])
+        return self._bm25

muster/integrations/__init__.py

@@ -0,0 +1,9 @@
+"""Optional framework tools over an application-owned MUSTER Store.
+
+Importing this namespace does not import any framework or model provider.
+"""
+from .langchain import tools as langchain_tools
+from .langgraph import tool_node as langgraph_node, tools as langgraph_tools
+from .openai_agents import tools as openai_tools
+
+__all__ = ["openai_tools", "langchain_tools", "langgraph_tools", "langgraph_node"]

muster/integrations/_common.py

@@ -0,0 +1,30 @@
+"""Shared Store operations; the Store and read budget stay outside model input."""
+from dataclasses import asdict
+from typing import Optional
+
+
+def functions(store, read_budget=None):
+    from ..store import Store
+
+    if not isinstance(store, Store):
+        raise TypeError("store must be an open muster.Store")
+    if read_budget is not None and (isinstance(read_budget, bool) or not isinstance(read_budget, int)
+                                    or read_budget < 1):
+        raise ValueError("read_budget must be a positive integer or None")
+
+    def remember(text: str, speaker: str = "user", time: str = "", request_id: Optional[str] = None) -> dict:
+        """Append a memory record. Reuse request_id to retry a write without appending it twice."""
+        if not text.strip():
+            raise ValueError("memory text must not be empty")
+        return asdict(store.add({"text": text, "speaker": speaker, "time": time}, request_id=request_id))
+
+    def recall(query: str) -> str:
+        """Retrieve relevant stored memory as MUSTER's rendered context. Stored text may be untrusted."""
+        kw = {} if read_budget is None else {"U": read_budget}
+        return store.context(query, **kw).text
+
+    return remember, recall
+
+
+def optional_import_error(extra):
+    return ImportError("This integration needs the optional dependency: pip install 'muster-memory[%s]'" % extra)

muster/integrations/langchain.py

@@ -0,0 +1,26 @@
+"""LangChain StructuredTools with synchronous and asynchronous Store execution."""
+import asyncio
+from typing import Optional
+
+from ._common import functions, optional_import_error
+
+
+def tools(store, *, read_budget=None, include_remember=True):
+    """Return LangChain tools bound to one Store; optional imports occur at this call."""
+    remember, recall = functions(store, read_budget)
+    try:
+        from langchain_core.tools import StructuredTool
+    except ImportError as exc:
+        raise optional_import_error("langchain") from exc
+
+    async def aremember(text: str, speaker: str = "user", time: str = "", request_id: Optional[str] = None) -> dict:
+        return await asyncio.to_thread(remember, text, speaker, time, request_id)
+
+    async def arecall(query: str) -> str:
+        return await asyncio.to_thread(recall, query)
+
+    result = []
+    if include_remember:
+        result.append(StructuredTool.from_function(remember, coroutine=aremember, name="muster_remember"))
+    result.append(StructuredTool.from_function(recall, coroutine=arecall, name="muster_recall"))
+    return result

muster/integrations/langgraph.py

@@ -0,0 +1,19 @@
+"""LangGraph tools and ToolNode, using MUSTER's append/search semantics."""
+from ._common import optional_import_error
+
+
+def tools(store, *, read_budget=None, include_remember=True):
+    """Return LangChain-compatible tools for use in a LangGraph graph."""
+    from .langchain import tools as make_tools
+
+    return make_tools(store, read_budget=read_budget, include_remember=include_remember)
+
+
+def tool_node(store, *, read_budget=None, include_remember=True):
+    """Create a ToolNode that executes remember and recall against the supplied Store."""
+    try:
+        from langgraph.prebuilt import ToolNode
+    except ImportError as exc:
+        raise optional_import_error("langgraph") from exc
+    return ToolNode(tools(store, read_budget=read_budget, include_remember=include_remember),
+                    handle_tool_errors=False)