muster-memory 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- muster/__init__.py +27 -0
- muster/_version.py +1 -0
- muster/adapters.py +33 -0
- muster/bpe.py +130 -0
- muster/cache.py +68 -0
- muster/cli.py +248 -0
- muster/compat/__init__.py +1 -0
- muster/compat/bundles.py +145 -0
- muster/compat/ce_cache.py +47 -0
- muster/compat/env.py +39 -0
- muster/compat/intermediates.py +140 -0
- muster/compat/recover.py +75 -0
- muster/compat/verify.py +142 -0
- muster/cut.py +291 -0
- muster/derive.py +146 -0
- muster/errors.py +39 -0
- muster/index.py +102 -0
- muster/integrations/__init__.py +9 -0
- muster/integrations/_common.py +30 -0
- muster/integrations/langchain.py +26 -0
- muster/integrations/langgraph.py +19 -0
- muster/integrations/openai_agents.py +32 -0
- muster/io.py +33 -0
- muster/memory.py +254 -0
- muster/models.py +128 -0
- muster/plan.py +75 -0
- muster/presets/__init__.py +43 -0
- muster/presets/paper.json +1182 -0
- muster/record_serial.py +90 -0
- muster/records.py +120 -0
- muster/render.py +89 -0
- muster/search.py +328 -0
- muster/serve.py +158 -0
- muster/source_order.py +197 -0
- muster/stats.py +108 -0
- muster/store.py +186 -0
- muster_memory-0.1.0.dist-info/METADATA +203 -0
- muster_memory-0.1.0.dist-info/RECORD +42 -0
- muster_memory-0.1.0.dist-info/WHEEL +4 -0
- muster_memory-0.1.0.dist-info/entry_points.txt +2 -0
- muster_memory-0.1.0.dist-info/licenses/LICENSE +202 -0
- muster_memory-0.1.0.dist-info/licenses/NOTICE +6 -0
muster/serve.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Connect MUSTER to agents and devices: an MCP server (Claude, Cursor, any MCP client) and a small HTTP JSON API
|
|
2
|
+
(any app, service or robot that can POST). Both wrap one Store file.
|
|
3
|
+
|
|
4
|
+
muster mcp --store agent.muster # stdio MCP server
|
|
5
|
+
muster http --store agent.muster --port 8765 # POST /v1/add, /v1/context, /v1/search; GET /v1/info
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import threading
|
|
9
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
10
|
+
|
|
11
|
+
from .store import Store
|
|
12
|
+
|
|
13
|
+
INSTRUCTIONS = ("Long-term memory that keeps every word. Call `remember` for anything worth keeping (one fact, "
|
|
14
|
+
"message or observation per call, with who said it and when if known). Call `recall` with a question "
|
|
15
|
+
"before answering anything that depends on the past; it returns the stored passages, grouped by the "
|
|
16
|
+
"part of the history they come from.")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _blocks(ctx, mem):
|
|
20
|
+
out = []
|
|
21
|
+
for b in ctx.blocks:
|
|
22
|
+
seen, recs = set(), []
|
|
23
|
+
for i in b.ids:
|
|
24
|
+
for r in mem.unit_records(i):
|
|
25
|
+
if id(r) not in seen: # records hold a dict (meta), so they are not hashable
|
|
26
|
+
seen.add(id(r))
|
|
27
|
+
recs.append({"id": r.id, "time": r.time, "speaker": r.speaker})
|
|
28
|
+
out.append({"index": b.index, "parent": str(b.parent), "ids": list(b.ids), "text": b.text, "records": recs})
|
|
29
|
+
return out
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Service:
|
|
33
|
+
"""The operations both servers expose, over one Store (thread-safe)."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, store: Store, *, allow_declared_records=False):
|
|
36
|
+
self.store = store
|
|
37
|
+
self.allow_declared_records = allow_declared_records
|
|
38
|
+
self.lock = threading.Lock()
|
|
39
|
+
|
|
40
|
+
def add(self, records, request_id=None):
|
|
41
|
+
with self.lock:
|
|
42
|
+
r = self.store.add(records, request_id=request_id)
|
|
43
|
+
return {"version": r.version, "duplicate": r.duplicate, "n_records": r.n_records}
|
|
44
|
+
|
|
45
|
+
def context(self, query, units=0):
|
|
46
|
+
with self.lock:
|
|
47
|
+
mem = self.store.head()
|
|
48
|
+
ctx = self.store.context(query, U=units or None)
|
|
49
|
+
return {"text": ctx.text, "ctxhash": ctx.ctxhash, "style": ctx.style, "ids": list(ctx.ids),
|
|
50
|
+
"blocks": _blocks(ctx, mem), "version": mem.version}
|
|
51
|
+
|
|
52
|
+
def search(self, query, units=0):
|
|
53
|
+
with self.lock:
|
|
54
|
+
mem = self.store.head()
|
|
55
|
+
sel = self.store.search(query, U=units or None)
|
|
56
|
+
ctx = mem.render(sel)
|
|
57
|
+
return {"ids": list(sel.ids), "units": [mem.units[i] for i in sel.ids], "parents": [str(mem.parent[i]) for i in sel.ids],
|
|
58
|
+
"blocks": _blocks(ctx, mem), "version": mem.version, "U": sel.U, "floors": list(sel.floors)}
|
|
59
|
+
|
|
60
|
+
def info(self):
|
|
61
|
+
with self.lock:
|
|
62
|
+
mem = self.store.head()
|
|
63
|
+
p = mem.plan
|
|
64
|
+
return {"version": mem.version, "units": len(mem), "parents": len(set(mem.parent)), "branch": p.branch,
|
|
65
|
+
"atom": p.atom, "chunk": p.chunk, "mscale": p.mscale, "U": p.U, "policy": p.policy, "render": p.render,
|
|
66
|
+
"mode": p.mode}
|
|
67
|
+
|
|
68
|
+
def declared_context(self, envelope, query, units=0):
|
|
69
|
+
"""One opt-in, stateless record-preserving context; does not append to Store."""
|
|
70
|
+
if not self.allow_declared_records:
|
|
71
|
+
raise ValueError("declared records endpoint is disabled; enable it explicitly")
|
|
72
|
+
from .source_order import build_declared
|
|
73
|
+
with self.lock:
|
|
74
|
+
mem = build_declared(envelope, models=self.store._models())
|
|
75
|
+
ctx = mem.context(query, U=units or None)
|
|
76
|
+
return {"text": ctx.text, "ctxhash": ctx.ctxhash, "style": ctx.style,
|
|
77
|
+
"ids": list(ctx.ids), "blocks": _blocks(ctx, mem),
|
|
78
|
+
"record_order": mem.ordered_records.diagnostics(), "stateless": True}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def mcp_server(service: Service, name="muster"):
|
|
82
|
+
try:
|
|
83
|
+
from mcp.server.mcpserver import MCPServer # mcp >= 2
|
|
84
|
+
except ImportError:
|
|
85
|
+
try:
|
|
86
|
+
from mcp.server.fastmcp import FastMCP as MCPServer # mcp 1.x
|
|
87
|
+
except ImportError:
|
|
88
|
+
raise SystemExit("the MCP server needs the MCP SDK: pip install 'muster-memory[mcp]'")
|
|
89
|
+
srv = MCPServer(name, instructions=INSTRUCTIONS)
|
|
90
|
+
|
|
91
|
+
@srv.tool(description="Store one thing to remember: a fact, a message or an observation. Optional: who said or saw "
|
|
92
|
+
"it (speaker), when (time, ISO 8601), and a session or episode id.")
|
|
93
|
+
def remember(text: str, speaker: str = "", time: str = "", session: str = "") -> dict:
|
|
94
|
+
rec = {"text": text}
|
|
95
|
+
for k, v in (("speaker", speaker), ("time", time), ("session", session)):
|
|
96
|
+
if v:
|
|
97
|
+
rec[k] = v
|
|
98
|
+
return service.add([rec])
|
|
99
|
+
|
|
100
|
+
@srv.tool(description="Store several records at once. Each record: {text, speaker?, time?, session?, id?}. "
|
|
101
|
+
"A request_id makes a retried call safe: it is applied once.")
|
|
102
|
+
def remember_many(records: list, request_id: str = "") -> dict:
|
|
103
|
+
return service.add(records, request_id=request_id or None)
|
|
104
|
+
|
|
105
|
+
@srv.tool(description="Recall what memory holds about a question: the stored passages, verbatim, grouped by where "
|
|
106
|
+
"they come from. `units` overrides how many passages to read (default: the memory's own).")
|
|
107
|
+
def recall(query: str, units: int = 0) -> str:
|
|
108
|
+
return service.context(query, units)["text"]
|
|
109
|
+
|
|
110
|
+
@srv.tool(description="Like recall, but structured: passage ids, texts and their groups.")
|
|
111
|
+
def search_memory(query: str, units: int = 0) -> dict:
|
|
112
|
+
return service.search(query, units)
|
|
113
|
+
|
|
114
|
+
@srv.tool(description="How this memory is organised: size, how it is cut and grouped, how much a recall reads.")
|
|
115
|
+
def memory_info() -> dict:
|
|
116
|
+
return service.info()
|
|
117
|
+
|
|
118
|
+
return srv
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def http_server(service: Service, host="127.0.0.1", port=8765):
|
|
122
|
+
class H(BaseHTTPRequestHandler):
|
|
123
|
+
def _send(self, code, obj):
|
|
124
|
+
body = json.dumps(obj, ensure_ascii=False).encode()
|
|
125
|
+
self.send_response(code)
|
|
126
|
+
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
127
|
+
self.send_header("Content-Length", str(len(body)))
|
|
128
|
+
self.end_headers()
|
|
129
|
+
self.wfile.write(body)
|
|
130
|
+
|
|
131
|
+
def log_message(self, *a):
|
|
132
|
+
pass
|
|
133
|
+
|
|
134
|
+
def do_GET(self):
|
|
135
|
+
if self.path in ("/health", "/v1/health"):
|
|
136
|
+
return self._send(200, {"ok": True})
|
|
137
|
+
if self.path == "/v1/info":
|
|
138
|
+
return self._send(200, service.info())
|
|
139
|
+
self._send(404, {"error": "not found"})
|
|
140
|
+
|
|
141
|
+
def do_POST(self):
|
|
142
|
+
try:
|
|
143
|
+
n = int(self.headers.get("Content-Length") or 0)
|
|
144
|
+
req = json.loads(self.rfile.read(n) or b"{}")
|
|
145
|
+
if self.path == "/v1/add":
|
|
146
|
+
recs = req.get("records") if "records" in req else [req]
|
|
147
|
+
return self._send(200, service.add(recs, request_id=req.get("request_id")))
|
|
148
|
+
if self.path == "/v1/context":
|
|
149
|
+
return self._send(200, service.context(req["query"], int(req.get("units") or 0)))
|
|
150
|
+
if self.path == "/v1/search":
|
|
151
|
+
return self._send(200, service.search(req["query"], int(req.get("units") or 0)))
|
|
152
|
+
if self.path == "/v1/declared-context":
|
|
153
|
+
return self._send(200, service.declared_context(req["store"], req["query"],
|
|
154
|
+
int(req.get("units") or 0)))
|
|
155
|
+
return self._send(404, {"error": "not found"})
|
|
156
|
+
except (KeyError, ValueError, TypeError) as e:
|
|
157
|
+
return self._send(400, {"error": "%s: %s" % (type(e).__name__, e)})
|
|
158
|
+
return ThreadingHTTPServer((host, port), H)
|
muster/source_order.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Opt-in record boundaries and source order, without inferred revision semantics.
|
|
2
|
+
|
|
3
|
+
The producer declares the source sequence. Prefix formats are explicit codecs,
|
|
4
|
+
not detectors; dates, gaps and repeated serials never trigger replacement.
|
|
5
|
+
"""
|
|
6
|
+
import copy
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
from dataclasses import asdict, dataclass
|
|
11
|
+
from typing import Optional, Tuple
|
|
12
|
+
|
|
13
|
+
from .errors import RecordError
|
|
14
|
+
from .stats import LEAD
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class SourceOrder:
|
|
19
|
+
source_sha256: str
|
|
20
|
+
source_locator: str
|
|
21
|
+
kind: str
|
|
22
|
+
declaration: str
|
|
23
|
+
namespace: str
|
|
24
|
+
|
|
25
|
+
def __post_init__(self):
|
|
26
|
+
if not isinstance(self.source_sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", self.source_sha256):
|
|
27
|
+
raise RecordError("source_sha256 must identify the original source version")
|
|
28
|
+
if self.kind not in ("structured_array", "explicit_append_sequence"):
|
|
29
|
+
raise RecordError("declared source order kind must be structured_array or explicit_append_sequence")
|
|
30
|
+
if any(not isinstance(v, str) or not v.strip()
|
|
31
|
+
for v in (self.source_locator, self.declaration, self.namespace)):
|
|
32
|
+
raise RecordError("source locator, order declaration and namespace are required")
|
|
33
|
+
try:
|
|
34
|
+
for value in (self.source_locator, self.declaration, self.namespace):
|
|
35
|
+
value.encode("utf-8")
|
|
36
|
+
except UnicodeError as exc:
|
|
37
|
+
raise RecordError("source provenance must be valid UTF-8 text") from exc
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class OrderedRecord:
|
|
42
|
+
id: str
|
|
43
|
+
occurrence_id: str
|
|
44
|
+
source_position: int
|
|
45
|
+
body: str
|
|
46
|
+
raw_text: str
|
|
47
|
+
prefix: str
|
|
48
|
+
marker: Optional[int]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class OrderedRecords:
|
|
53
|
+
records: Tuple[OrderedRecord, ...]
|
|
54
|
+
order: SourceOrder
|
|
55
|
+
record_format: str
|
|
56
|
+
_original_json: str
|
|
57
|
+
|
|
58
|
+
def roundtrip(self):
|
|
59
|
+
"""Original JSON values, including original prefixes and all metadata."""
|
|
60
|
+
return json.loads(self._original_json)
|
|
61
|
+
|
|
62
|
+
def diagnostics(self):
|
|
63
|
+
markers = [r.marker for r in self.records]
|
|
64
|
+
present = bool(markers) and all(m is not None for m in markers)
|
|
65
|
+
return dict(feature="declared_order_records_v1", experimental=True,
|
|
66
|
+
source_order=asdict(self.order), record_format=self.record_format,
|
|
67
|
+
input_records_sha256=hashlib.sha256(self._original_json.encode("utf-8")).hexdigest(),
|
|
68
|
+
record_ids=[r.id for r in self.records],
|
|
69
|
+
occurrence_ids=[r.occurrence_id for r in self.records],
|
|
70
|
+
marker_values=markers,
|
|
71
|
+
contiguous_markers=bool(present and all(b == a + 1 for a, b in zip(markers, markers[1:]))),
|
|
72
|
+
source_order_preserved=True, marker_is_revision_clock=False,
|
|
73
|
+
supersession_eligible=False, deleted_record_ids=[], policy="keep_all")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
_PREFIX = {"step": re.compile(r"^(\[step (\d+)\] )(.*)$", re.DOTALL),
|
|
77
|
+
"integer_dot": re.compile(r"^((\d+)\. )(.*)$", re.DOTALL)}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def normalize_ordered_records(records, order, *, record_format="content"):
|
|
81
|
+
"""Preserve each declared record as one body, with reversible prefix parsing.
|
|
82
|
+
|
|
83
|
+
`content` accepts content/text mappings or strings. `step` and `integer_dot`
|
|
84
|
+
accept the same explicit record sequence with a producer-declared prefix.
|
|
85
|
+
`serial` reads the shared leading-serial lexer without detector thresholds;
|
|
86
|
+
its marker is diagnostic and the declared array remains the source order.
|
|
87
|
+
A single plain-text blob supplies neither trustworthy boundaries nor order.
|
|
88
|
+
The source hash is a caller-supplied provenance assertion; this function
|
|
89
|
+
also records a canonical hash of the received JSON records.
|
|
90
|
+
"""
|
|
91
|
+
if isinstance(order, dict):
|
|
92
|
+
try:
|
|
93
|
+
order = SourceOrder(**order)
|
|
94
|
+
except TypeError as exc:
|
|
95
|
+
raise RecordError("invalid source order fields") from exc
|
|
96
|
+
if not isinstance(order, SourceOrder):
|
|
97
|
+
raise RecordError("explicit SourceOrder provenance is required")
|
|
98
|
+
if record_format not in ("content", "step", "integer_dot", "serial"):
|
|
99
|
+
raise RecordError("record_format must be content, step, integer_dot or serial")
|
|
100
|
+
if not isinstance(records, (list, tuple)) or not records:
|
|
101
|
+
raise RecordError("explicit nonempty record array required; plain text has no declared boundaries")
|
|
102
|
+
try:
|
|
103
|
+
original = json.dumps(records, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
|
104
|
+
original.encode("utf-8")
|
|
105
|
+
except (TypeError, ValueError, UnicodeError) as exc:
|
|
106
|
+
raise RecordError("declared records must be valid UTF-8 JSON values") from exc
|
|
107
|
+
out, seen = [], set()
|
|
108
|
+
for i, row in enumerate(copy.deepcopy(records)):
|
|
109
|
+
occurrence = "%s@%s#record%d" % (order.namespace, order.source_sha256, i)
|
|
110
|
+
if isinstance(row, dict):
|
|
111
|
+
raw = row.get("text", row.get("content"))
|
|
112
|
+
if "text" in row and "content" in row and row["text"] != row["content"]:
|
|
113
|
+
raise RecordError("text and content disagree in record %d" % i)
|
|
114
|
+
rid = row.get("id", occurrence)
|
|
115
|
+
else:
|
|
116
|
+
raw, rid = row, occurrence
|
|
117
|
+
if not isinstance(raw, str):
|
|
118
|
+
raise RecordError("record %d needs a content/text string" % i)
|
|
119
|
+
if not isinstance(rid, (str, int)) or isinstance(rid, bool) or not str(rid).strip():
|
|
120
|
+
raise RecordError("record %d has an invalid ID" % i)
|
|
121
|
+
rid = str(rid)
|
|
122
|
+
if rid in seen:
|
|
123
|
+
raise RecordError("duplicate record ID %r" % rid)
|
|
124
|
+
seen.add(rid)
|
|
125
|
+
prefix, body, marker = "", raw, None
|
|
126
|
+
if record_format == "serial":
|
|
127
|
+
m = LEAD.match(raw)
|
|
128
|
+
if m is None or m.end(2) > 32:
|
|
129
|
+
raise RecordError("record %d does not match declared serial prefix" % i)
|
|
130
|
+
try:
|
|
131
|
+
marker = int(m.group(2))
|
|
132
|
+
except ValueError as exc:
|
|
133
|
+
raise RecordError("record %d has an unsupported serial integer" % i) from exc
|
|
134
|
+
prefix, body = raw[:m.end()], raw[m.end():]
|
|
135
|
+
elif record_format != "content":
|
|
136
|
+
m = _PREFIX[record_format].fullmatch(raw)
|
|
137
|
+
if m is None:
|
|
138
|
+
raise RecordError("record %d does not match declared %s prefix" % (i, record_format))
|
|
139
|
+
try:
|
|
140
|
+
marker = int(m.group(2))
|
|
141
|
+
except ValueError as exc:
|
|
142
|
+
raise RecordError("record %d has an unsupported serial integer" % i) from exc
|
|
143
|
+
prefix, body = m.group(1), m.group(3)
|
|
144
|
+
if not body.strip():
|
|
145
|
+
raise RecordError("record %d has an empty body" % i)
|
|
146
|
+
if prefix + body != raw:
|
|
147
|
+
raise AssertionError("record prefix roundtrip failed")
|
|
148
|
+
out.append(OrderedRecord(rid, occurrence, i, body, raw, prefix, marker))
|
|
149
|
+
return OrderedRecords(tuple(out), order, record_format, original)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def build_declared(envelope, *, models=None, embed=True):
|
|
153
|
+
"""Build the optional record-preserving view from one JSON envelope."""
|
|
154
|
+
if not isinstance(envelope, dict) or set(envelope) - {"records", "order", "format"}:
|
|
155
|
+
raise RecordError("declared envelope has records, order and optional format only")
|
|
156
|
+
try:
|
|
157
|
+
records, order = envelope["records"], envelope["order"]
|
|
158
|
+
except KeyError as exc:
|
|
159
|
+
raise RecordError("declared envelope needs records and order") from exc
|
|
160
|
+
from .memory import build
|
|
161
|
+
return build(records, models=models, embed=embed, declared_order=order,
|
|
162
|
+
record_format=envelope.get("format", "content"))
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def build_ordered_memory(records, order, *, record_format, plan, models, embed):
|
|
166
|
+
from . import bpe
|
|
167
|
+
from .cut import Cut
|
|
168
|
+
from .memory import Memory, resolve_plan
|
|
169
|
+
from .records import Record
|
|
170
|
+
normalized = normalize_ordered_records(records, order, record_format=record_format)
|
|
171
|
+
bodies = tuple(r.body for r in normalized.records)
|
|
172
|
+
text, spans, pos = "\n\n".join(bodies), [], 0
|
|
173
|
+
for body in bodies:
|
|
174
|
+
spans.append((pos, pos + len(body)))
|
|
175
|
+
pos += len(body) + 2
|
|
176
|
+
p = resolve_plan(text, plan)
|
|
177
|
+
if p.mode != "native":
|
|
178
|
+
raise RecordError("declared-order records are a native opt-in mode; exact replay is unchanged")
|
|
179
|
+
trace = ("Caller declares record boundaries and source-array/append order; no date sorting.",
|
|
180
|
+
"One body per unit, one parent per occurrence; all versions kept without a revision contract.",
|
|
181
|
+
"The base plan supplies U=%d; declared record granularity replaces automatic token packing." % p.U)
|
|
182
|
+
p = p.with_(branch="record", atom=None, chunk=None, mscale=0, policy="flrs", render="r4", supersede="off",
|
|
183
|
+
compat=None, source="declared-order:record-boundaries-v1",
|
|
184
|
+
trace=trace)
|
|
185
|
+
diagnostics = normalized.diagnostics()
|
|
186
|
+
c = Cut(bodies, bodies, tuple("u:" + r.occurrence_id for r in normalized.records),
|
|
187
|
+
tuple(range(len(bodies))), tuple(spans),
|
|
188
|
+
tuple(len(bpe.o200k().encode(b, disallowed_special=())) for b in bodies), None, diagnostics)
|
|
189
|
+
recs = [Record(text=r.body, id=r.id, meta={"occurrence_id": r.occurrence_id,
|
|
190
|
+
"source_position": str(r.source_position)}) for r in normalized.records]
|
|
191
|
+
mem = Memory(plan=p, text=text, cut=c, records=recs, record_spans=spans, models=models,
|
|
192
|
+
ordered_records=normalized)
|
|
193
|
+
if mem.index.structural or mem.index.newest_row:
|
|
194
|
+
raise AssertionError("source order unexpectedly enabled revision pruning")
|
|
195
|
+
if embed:
|
|
196
|
+
mem.embed(models)
|
|
197
|
+
return mem
|
muster/stats.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""The two statistics that decide a store's representation, computed from its text with no model.
|
|
2
|
+
|
|
3
|
+
serial fraction the share of non-empty lines (first 20,000) that open with "<integer>. <body>"
|
|
4
|
+
record share the token-weighted share of blank-line records of at most the reference chunk
|
|
5
|
+
|
|
6
|
+
`store_stats` is scripts/derive.py:69-79 (one store, CRLF normalized, special-token strings read as text).
|
|
7
|
+
`collection_stats` is harness/derive_config.py:43-64 (the first rows of a benchmark cell, pooled), which the paper's
|
|
8
|
+
per-cell decisions used.
|
|
9
|
+
|
|
10
|
+
`lead_serial` is not in the paper: it finds a log that numbers its records in its own style ("Step 3: ..."),
|
|
11
|
+
and only `derive` reads it, after the serial fraction has failed.
|
|
12
|
+
"""
|
|
13
|
+
import re
|
|
14
|
+
import statistics
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
|
|
17
|
+
from . import bpe
|
|
18
|
+
from .records import normalize_newlines
|
|
19
|
+
|
|
20
|
+
NUM = re.compile(r"^\s*\d+\.\s+\S")
|
|
21
|
+
MAX_LINES = 20000
|
|
22
|
+
# a serial under the log's own spelling: an optional bracket and word, the integer, an optional closer, then the body
|
|
23
|
+
# ("Step 3: ", "#3 ", "(3) ", "[3] ", "3. "). Thresholds set after the paper (2026-11) on public benchmark logs.
|
|
24
|
+
LEAD = re.compile(r"^(\s*[\[(#]?\s*(?:[A-Za-z]+\s*)?)(\d{1,8})([\]).:]?\s+)(?=\S)")
|
|
25
|
+
LEAD_TAU, LEAD_LINE_TOKENS = 0.95, 20
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Stats:
|
|
30
|
+
serial: float
|
|
31
|
+
record_share: float
|
|
32
|
+
tokens: int # store: tokens in blank-line records; collection: mean whole-row tokens
|
|
33
|
+
lines_used: int
|
|
34
|
+
rows: int = 1
|
|
35
|
+
scope: str = "store"
|
|
36
|
+
|
|
37
|
+
def to_dict(self):
|
|
38
|
+
return dict(serial=self.serial, record_share=self.record_share, tokens=self.tokens,
|
|
39
|
+
lines_used=self.lines_used, rows=self.rows, scope=self.scope)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _serial(text):
|
|
43
|
+
lines = [ln for ln in text.split("\n") if ln.strip()][:MAX_LINES]
|
|
44
|
+
return (sum(1 for ln in lines if NUM.match(ln)) / float(len(lines)) if lines else 0.0), len(lines)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def lead_match(s):
|
|
48
|
+
"""LEAD at the start of `s` when its integer ends within the first 32 characters, else None."""
|
|
49
|
+
m = LEAD.match(s)
|
|
50
|
+
return m if m and m.end(2) <= 32 else None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def lead_serial(text):
|
|
54
|
+
"""(share of lines after the first, among the first 20,000 non-empty lines, whose leading serial is the previous
|
|
55
|
+
line's plus one, median cl100k tokens per line). The median is counted only when the share reaches LEAD_TAU, else
|
|
56
|
+
it is None. A serial must end within a line's first 32 characters."""
|
|
57
|
+
lines = [ln for ln in normalize_newlines(text).split("\n") if ln.strip()][:MAX_LINES]
|
|
58
|
+
if not lines:
|
|
59
|
+
return 0.0, None
|
|
60
|
+
prev, step = None, 0
|
|
61
|
+
for ln in lines:
|
|
62
|
+
m = lead_match(ln)
|
|
63
|
+
v = int(m.group(2)) if m else None
|
|
64
|
+
if v is not None and prev is not None and v == prev + 1:
|
|
65
|
+
step += 1
|
|
66
|
+
prev = v
|
|
67
|
+
share = step / float(max(len(lines) - 1, 1))
|
|
68
|
+
if share < LEAD_TAU:
|
|
69
|
+
return share, None
|
|
70
|
+
enc = bpe.cl100k()
|
|
71
|
+
return share, statistics.median([bpe.count(enc, ln) for ln in lines])
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def store_stats(text: str, reference_chunk: int = 4096) -> Stats:
|
|
75
|
+
enc = bpe.cl100k()
|
|
76
|
+
text = normalize_newlines(text)
|
|
77
|
+
total = fit = 0
|
|
78
|
+
for rec in text.split("\n\n"):
|
|
79
|
+
if rec.strip():
|
|
80
|
+
t = bpe.count(enc, rec)
|
|
81
|
+
total += t
|
|
82
|
+
fit += t if t <= reference_chunk else 0
|
|
83
|
+
serial, n = _serial(text)
|
|
84
|
+
return Stats(serial, (fit / float(total) if total else 0.0), total, n)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def collection_stats(texts, reference_chunk: int, *, sample: int = 3, mode: str = "native") -> Stats:
|
|
88
|
+
enc = bpe.cl100k()
|
|
89
|
+
enc_count = (lambda s: len(enc.encode(s))) if mode == "exact" else (lambda s: bpe.count(enc, s))
|
|
90
|
+
tot = fit = 0
|
|
91
|
+
fracs, toks, used = [], [], 0
|
|
92
|
+
rows = list(texts)[:sample]
|
|
93
|
+
for txt in rows:
|
|
94
|
+
txt = str(txt)
|
|
95
|
+
toks.append(enc_count(txt))
|
|
96
|
+
for p in txt.split("\n\n"):
|
|
97
|
+
if not p.strip():
|
|
98
|
+
continue
|
|
99
|
+
t = enc_count(p)
|
|
100
|
+
tot += t
|
|
101
|
+
if t <= reference_chunk:
|
|
102
|
+
fit += t
|
|
103
|
+
lines = [ln for ln in txt.split("\n") if ln.strip()][:MAX_LINES]
|
|
104
|
+
used += len(lines)
|
|
105
|
+
if lines:
|
|
106
|
+
fracs.append(sum(1 for ln in lines if NUM.match(ln)) / float(len(lines)))
|
|
107
|
+
return Stats(sum(fracs) / len(fracs) if fracs else 0.0, fit / float(tot) if tot else 0.0,
|
|
108
|
+
(sum(toks) // len(toks)) if toks else 0, used, rows=len(rows), scope="collection")
|
muster/store.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Store: a growing memory in one SQLite file, for services, agents and robots.
|
|
2
|
+
|
|
3
|
+
store = muster.Store.open("agent.muster")
|
|
4
|
+
store.add([{"speaker": "user", "text": "I moved to Lisbon in May.", "time": "2026-05-02"}])
|
|
5
|
+
print(store.context("Where does the user live now?").text)
|
|
6
|
+
|
|
7
|
+
`add` only appends records (one transaction, no model call), so it is fast and safe to call from a hot path. The
|
|
8
|
+
first search after an add builds the new version: the grown store is cut, indexed and planned exactly as a fresh
|
|
9
|
+
build of all its records would be, and only keys never embedded before are sent to the encoder. Reranker scores and
|
|
10
|
+
vectors are cached by text, so they survive restarts and growth.
|
|
11
|
+
"""
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import sqlite3
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
from dataclasses import MISSING, dataclass, fields
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Optional
|
|
20
|
+
|
|
21
|
+
from .cache import ScoreCache, VectorCache, model_key
|
|
22
|
+
from .errors import StoreLocked
|
|
23
|
+
from .memory import Memory, build
|
|
24
|
+
from .records import Record, as_record
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class AddResult:
|
|
29
|
+
version: int
|
|
30
|
+
duplicate: bool
|
|
31
|
+
n_records: int
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _rec_json(r: Record):
|
|
35
|
+
d = {}
|
|
36
|
+
for f in fields(Record):
|
|
37
|
+
v = getattr(r, f.name)
|
|
38
|
+
default = f.default_factory() if f.default_factory is not MISSING else f.default
|
|
39
|
+
if f.name == "text" or v != default:
|
|
40
|
+
d[f.name] = dict(v) if f.name == "meta" else (list(v) if isinstance(v, tuple) else v)
|
|
41
|
+
return json.dumps(d, ensure_ascii=False, sort_keys=True)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Store:
|
|
45
|
+
def __init__(self, path, *, models=None, plan=None, template=None, readonly=False):
|
|
46
|
+
self.path, self.models, self.plan, self.template, self.readonly = os.fsdecode(path), models, plan, template, readonly
|
|
47
|
+
self._lock = threading.RLock()
|
|
48
|
+
# URI escaping preserves literal '?', '#' and '%' in application paths.
|
|
49
|
+
# SQLite enforces read-only access and refuses to create a missing file.
|
|
50
|
+
target = Path(self.path).absolute().as_uri() + "?mode=ro" if readonly else self.path
|
|
51
|
+
self.db = sqlite3.connect(target, uri=readonly, check_same_thread=False, isolation_level=None)
|
|
52
|
+
if not readonly:
|
|
53
|
+
self.db.execute("PRAGMA journal_mode=WAL")
|
|
54
|
+
self.db.execute("PRAGMA synchronous=FULL")
|
|
55
|
+
self.db.execute("CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT)")
|
|
56
|
+
self.db.execute("CREATE TABLE IF NOT EXISTS records(seq INTEGER PRIMARY KEY, request_id TEXT, record TEXT)")
|
|
57
|
+
self.db.execute("CREATE TABLE IF NOT EXISTS requests(request_id TEXT PRIMARY KEY, version INT)")
|
|
58
|
+
self._head: Optional[Memory] = None
|
|
59
|
+
self._head_version = -1
|
|
60
|
+
self._scores = None
|
|
61
|
+
self._vectors = None
|
|
62
|
+
if not readonly:
|
|
63
|
+
pid = self.db.execute("SELECT v FROM meta WHERE k='writer'").fetchone()
|
|
64
|
+
if pid and pid[0] != str(os.getpid()) and _alive(int(pid[0])):
|
|
65
|
+
raise StoreLocked("%s is open for writing by process %s" % (path, pid[0]))
|
|
66
|
+
self.db.execute("INSERT OR REPLACE INTO meta VALUES ('writer', ?)", (str(os.getpid()),))
|
|
67
|
+
|
|
68
|
+
@classmethod
|
|
69
|
+
def open(cls, path, **kw):
|
|
70
|
+
return cls(path, **kw)
|
|
71
|
+
|
|
72
|
+
# ------------------------------------------------------------- writing
|
|
73
|
+
def add(self, records, *, request_id: Optional[str] = None) -> AddResult:
|
|
74
|
+
"""Append records. A repeated request_id is acknowledged once and appends nothing."""
|
|
75
|
+
if self.readonly:
|
|
76
|
+
raise PermissionError("store opened readonly")
|
|
77
|
+
if isinstance(records, (str, Record)) or hasattr(records, "keys"):
|
|
78
|
+
records = [records]
|
|
79
|
+
recs = [as_record(r) for r in records]
|
|
80
|
+
with self._lock:
|
|
81
|
+
cur = self.db.cursor()
|
|
82
|
+
cur.execute("BEGIN IMMEDIATE")
|
|
83
|
+
try:
|
|
84
|
+
if request_id is not None:
|
|
85
|
+
r = cur.execute("SELECT version FROM requests WHERE request_id=?", (request_id,)).fetchone()
|
|
86
|
+
if r is not None:
|
|
87
|
+
cur.execute("COMMIT")
|
|
88
|
+
return AddResult(r[0], True, 0)
|
|
89
|
+
cur.executemany("INSERT INTO records(request_id, record) VALUES (?, ?)",
|
|
90
|
+
[(request_id, _rec_json(x)) for x in recs])
|
|
91
|
+
v = cur.execute("SELECT COUNT(*) FROM records").fetchone()[0]
|
|
92
|
+
if request_id is not None:
|
|
93
|
+
cur.execute("INSERT INTO requests VALUES (?, ?)", (request_id, v))
|
|
94
|
+
cur.execute("COMMIT")
|
|
95
|
+
except Exception:
|
|
96
|
+
cur.execute("ROLLBACK")
|
|
97
|
+
raise
|
|
98
|
+
return AddResult(v, False, len(recs))
|
|
99
|
+
|
|
100
|
+
# ------------------------------------------------------------- reading
|
|
101
|
+
@property
|
|
102
|
+
def version(self):
|
|
103
|
+
return self.db.execute("SELECT COUNT(*) FROM records").fetchone()[0]
|
|
104
|
+
|
|
105
|
+
def records(self):
|
|
106
|
+
return [as_record(json.loads(r[0])) for r in self.db.execute("SELECT record FROM records ORDER BY seq")]
|
|
107
|
+
|
|
108
|
+
def _models(self):
|
|
109
|
+
if self.models is None:
|
|
110
|
+
from .models import Models
|
|
111
|
+
self.models = Models()
|
|
112
|
+
return self.models
|
|
113
|
+
|
|
114
|
+
def head(self) -> Memory:
|
|
115
|
+
"""The current version as a Memory, rebuilt only when records were added since the last call."""
|
|
116
|
+
with self._lock:
|
|
117
|
+
v = self.version
|
|
118
|
+
if self._head is not None and self._head_version == v:
|
|
119
|
+
return self._head
|
|
120
|
+
kw = {"template": self.template} if self.template else {}
|
|
121
|
+
mem = build(self.records(), self.plan, models=self._models(), embed=False, **kw)
|
|
122
|
+
mem.version = v
|
|
123
|
+
if len(mem):
|
|
124
|
+
self._embed(mem)
|
|
125
|
+
self._head, self._head_version = mem, v
|
|
126
|
+
return mem
|
|
127
|
+
|
|
128
|
+
def _embed(self, mem):
|
|
129
|
+
import numpy as np
|
|
130
|
+
m = self._models()
|
|
131
|
+
if self._vectors is None:
|
|
132
|
+
if self.readonly and not self.db.execute(
|
|
133
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='vectors'").fetchone():
|
|
134
|
+
self.db.execute("CREATE TEMP TABLE vectors(model TEXT, ksha BLOB, dim INT, vec BLOB, "
|
|
135
|
+
"PRIMARY KEY (model, ksha))")
|
|
136
|
+
self._vectors = VectorCache(self.db, model_key(m.encoder) + "|%d" % mem.plan.encoder.max_length,
|
|
137
|
+
create=not self.readonly)
|
|
138
|
+
keys = list(mem.cut.keys)
|
|
139
|
+
have = self._vectors.get_many(keys)
|
|
140
|
+
todo = [k for k in dict.fromkeys(keys) if k not in have]
|
|
141
|
+
if todo:
|
|
142
|
+
vec = m.encoder.encode(todo, batch_size=mem.plan.encoder.batch_size, normalize_embeddings=True)
|
|
143
|
+
new = {k: np.asarray(v, dtype=np.float32).tobytes() for k, v in zip(todo, vec)}
|
|
144
|
+
if not self.readonly: # a reader keeps new vectors in memory and writes nothing
|
|
145
|
+
self.db.execute("BEGIN")
|
|
146
|
+
self._vectors.put_many(new)
|
|
147
|
+
self.db.execute("COMMIT")
|
|
148
|
+
have.update(new)
|
|
149
|
+
mem.vecs = np.stack([np.frombuffer(have[k], dtype=np.float32) for k in keys])
|
|
150
|
+
|
|
151
|
+
def score_cache(self):
|
|
152
|
+
if self._scores is None:
|
|
153
|
+
self._scores = ScoreCache(None if (self.readonly or self.path == ":memory:") else self.path + ".scores",
|
|
154
|
+
model=model_key(self._models().reranker))
|
|
155
|
+
return self._scores
|
|
156
|
+
|
|
157
|
+
def search(self, query, **kw):
|
|
158
|
+
mem = self.head()
|
|
159
|
+
return mem.search(query, scores=self.score_cache(), models=self._models(), **kw)
|
|
160
|
+
|
|
161
|
+
def context(self, query, **kw):
|
|
162
|
+
mem = self.head()
|
|
163
|
+
style = kw.pop("style", None)
|
|
164
|
+
return mem.render(mem.search(query, scores=self.score_cache(), models=self._models(), **kw), style=style)
|
|
165
|
+
|
|
166
|
+
def close(self):
|
|
167
|
+
if not self.readonly:
|
|
168
|
+
try:
|
|
169
|
+
self.db.execute("DELETE FROM meta WHERE k='writer' AND v=?", (str(os.getpid()),))
|
|
170
|
+
except sqlite3.Error:
|
|
171
|
+
pass
|
|
172
|
+
self.db.close()
|
|
173
|
+
|
|
174
|
+
def __enter__(self):
|
|
175
|
+
return self
|
|
176
|
+
|
|
177
|
+
def __exit__(self, *a):
|
|
178
|
+
self.close()
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _alive(pid):
|
|
182
|
+
try:
|
|
183
|
+
os.kill(pid, 0)
|
|
184
|
+
return True
|
|
185
|
+
except OSError:
|
|
186
|
+
return False
|