muster-memory 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
muster/serve.py ADDED
@@ -0,0 +1,158 @@
1
+ """Connect MUSTER to agents and devices: an MCP server (Claude, Cursor, any MCP client) and a small HTTP JSON API
2
+ (any app, service or robot that can POST). Both wrap one Store file.
3
+
4
+ muster mcp --store agent.muster # stdio MCP server
5
+ muster http --store agent.muster --port 8765 # POST /v1/add, /v1/context, /v1/search; GET /v1/info
6
+ """
7
+ import json
8
+ import threading
9
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
10
+
11
+ from .store import Store
12
+
13
+ INSTRUCTIONS = ("Long-term memory that keeps every word. Call `remember` for anything worth keeping (one fact, "
14
+ "message or observation per call, with who said it and when if known). Call `recall` with a question "
15
+ "before answering anything that depends on the past; it returns the stored passages, grouped by the "
16
+ "part of the history they come from.")
17
+
18
+
19
+ def _blocks(ctx, mem):
20
+ out = []
21
+ for b in ctx.blocks:
22
+ seen, recs = set(), []
23
+ for i in b.ids:
24
+ for r in mem.unit_records(i):
25
+ if id(r) not in seen: # records hold a dict (meta), so they are not hashable
26
+ seen.add(id(r))
27
+ recs.append({"id": r.id, "time": r.time, "speaker": r.speaker})
28
+ out.append({"index": b.index, "parent": str(b.parent), "ids": list(b.ids), "text": b.text, "records": recs})
29
+ return out
30
+
31
+
32
+ class Service:
33
+ """The operations both servers expose, over one Store (thread-safe)."""
34
+
35
+ def __init__(self, store: Store, *, allow_declared_records=False):
36
+ self.store = store
37
+ self.allow_declared_records = allow_declared_records
38
+ self.lock = threading.Lock()
39
+
40
+ def add(self, records, request_id=None):
41
+ with self.lock:
42
+ r = self.store.add(records, request_id=request_id)
43
+ return {"version": r.version, "duplicate": r.duplicate, "n_records": r.n_records}
44
+
45
+ def context(self, query, units=0):
46
+ with self.lock:
47
+ mem = self.store.head()
48
+ ctx = self.store.context(query, U=units or None)
49
+ return {"text": ctx.text, "ctxhash": ctx.ctxhash, "style": ctx.style, "ids": list(ctx.ids),
50
+ "blocks": _blocks(ctx, mem), "version": mem.version}
51
+
52
+ def search(self, query, units=0):
53
+ with self.lock:
54
+ mem = self.store.head()
55
+ sel = self.store.search(query, U=units or None)
56
+ ctx = mem.render(sel)
57
+ return {"ids": list(sel.ids), "units": [mem.units[i] for i in sel.ids], "parents": [str(mem.parent[i]) for i in sel.ids],
58
+ "blocks": _blocks(ctx, mem), "version": mem.version, "U": sel.U, "floors": list(sel.floors)}
59
+
60
+ def info(self):
61
+ with self.lock:
62
+ mem = self.store.head()
63
+ p = mem.plan
64
+ return {"version": mem.version, "units": len(mem), "parents": len(set(mem.parent)), "branch": p.branch,
65
+ "atom": p.atom, "chunk": p.chunk, "mscale": p.mscale, "U": p.U, "policy": p.policy, "render": p.render,
66
+ "mode": p.mode}
67
+
68
+ def declared_context(self, envelope, query, units=0):
69
+ """One opt-in, stateless record-preserving context; does not append to Store."""
70
+ if not self.allow_declared_records:
71
+ raise ValueError("declared records endpoint is disabled; enable it explicitly")
72
+ from .source_order import build_declared
73
+ with self.lock:
74
+ mem = build_declared(envelope, models=self.store._models())
75
+ ctx = mem.context(query, U=units or None)
76
+ return {"text": ctx.text, "ctxhash": ctx.ctxhash, "style": ctx.style,
77
+ "ids": list(ctx.ids), "blocks": _blocks(ctx, mem),
78
+ "record_order": mem.ordered_records.diagnostics(), "stateless": True}
79
+
80
+
81
+ def mcp_server(service: Service, name="muster"):
82
+ try:
83
+ from mcp.server.mcpserver import MCPServer # mcp >= 2
84
+ except ImportError:
85
+ try:
86
+ from mcp.server.fastmcp import FastMCP as MCPServer # mcp 1.x
87
+ except ImportError:
88
+ raise SystemExit("the MCP server needs the MCP SDK: pip install 'muster-memory[mcp]'")
89
+ srv = MCPServer(name, instructions=INSTRUCTIONS)
90
+
91
+ @srv.tool(description="Store one thing to remember: a fact, a message or an observation. Optional: who said or saw "
92
+ "it (speaker), when (time, ISO 8601), and a session or episode id.")
93
+ def remember(text: str, speaker: str = "", time: str = "", session: str = "") -> dict:
94
+ rec = {"text": text}
95
+ for k, v in (("speaker", speaker), ("time", time), ("session", session)):
96
+ if v:
97
+ rec[k] = v
98
+ return service.add([rec])
99
+
100
+ @srv.tool(description="Store several records at once. Each record: {text, speaker?, time?, session?, id?}. "
101
+ "A request_id makes a retried call safe: it is applied once.")
102
+ def remember_many(records: list, request_id: str = "") -> dict:
103
+ return service.add(records, request_id=request_id or None)
104
+
105
+ @srv.tool(description="Recall what memory holds about a question: the stored passages, verbatim, grouped by where "
106
+ "they come from. `units` overrides how many passages to read (default: the memory's own).")
107
+ def recall(query: str, units: int = 0) -> str:
108
+ return service.context(query, units)["text"]
109
+
110
+ @srv.tool(description="Like recall, but structured: passage ids, texts and their groups.")
111
+ def search_memory(query: str, units: int = 0) -> dict:
112
+ return service.search(query, units)
113
+
114
+ @srv.tool(description="How this memory is organised: size, how it is cut and grouped, how much a recall reads.")
115
+ def memory_info() -> dict:
116
+ return service.info()
117
+
118
+ return srv
119
+
120
+
121
+ def http_server(service: Service, host="127.0.0.1", port=8765):
122
+ class H(BaseHTTPRequestHandler):
123
+ def _send(self, code, obj):
124
+ body = json.dumps(obj, ensure_ascii=False).encode()
125
+ self.send_response(code)
126
+ self.send_header("Content-Type", "application/json; charset=utf-8")
127
+ self.send_header("Content-Length", str(len(body)))
128
+ self.end_headers()
129
+ self.wfile.write(body)
130
+
131
+ def log_message(self, *a):
132
+ pass
133
+
134
+ def do_GET(self):
135
+ if self.path in ("/health", "/v1/health"):
136
+ return self._send(200, {"ok": True})
137
+ if self.path == "/v1/info":
138
+ return self._send(200, service.info())
139
+ self._send(404, {"error": "not found"})
140
+
141
+ def do_POST(self):
142
+ try:
143
+ n = int(self.headers.get("Content-Length") or 0)
144
+ req = json.loads(self.rfile.read(n) or b"{}")
145
+ if self.path == "/v1/add":
146
+ recs = req.get("records") if "records" in req else [req]
147
+ return self._send(200, service.add(recs, request_id=req.get("request_id")))
148
+ if self.path == "/v1/context":
149
+ return self._send(200, service.context(req["query"], int(req.get("units") or 0)))
150
+ if self.path == "/v1/search":
151
+ return self._send(200, service.search(req["query"], int(req.get("units") or 0)))
152
+ if self.path == "/v1/declared-context":
153
+ return self._send(200, service.declared_context(req["store"], req["query"],
154
+ int(req.get("units") or 0)))
155
+ return self._send(404, {"error": "not found"})
156
+ except (KeyError, ValueError, TypeError) as e:
157
+ return self._send(400, {"error": "%s: %s" % (type(e).__name__, e)})
158
+ return ThreadingHTTPServer((host, port), H)
muster/source_order.py ADDED
@@ -0,0 +1,197 @@
1
+ """Opt-in record boundaries and source order, without inferred revision semantics.
2
+
3
+ The producer declares the source sequence. Prefix formats are explicit codecs,
4
+ not detectors; dates, gaps and repeated serials never trigger replacement.
5
+ """
6
+ import copy
7
+ import hashlib
8
+ import json
9
+ import re
10
+ from dataclasses import asdict, dataclass
11
+ from typing import Optional, Tuple
12
+
13
+ from .errors import RecordError
14
+ from .stats import LEAD
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class SourceOrder:
19
+ source_sha256: str
20
+ source_locator: str
21
+ kind: str
22
+ declaration: str
23
+ namespace: str
24
+
25
+ def __post_init__(self):
26
+ if not isinstance(self.source_sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", self.source_sha256):
27
+ raise RecordError("source_sha256 must identify the original source version")
28
+ if self.kind not in ("structured_array", "explicit_append_sequence"):
29
+ raise RecordError("declared source order kind must be structured_array or explicit_append_sequence")
30
+ if any(not isinstance(v, str) or not v.strip()
31
+ for v in (self.source_locator, self.declaration, self.namespace)):
32
+ raise RecordError("source locator, order declaration and namespace are required")
33
+ try:
34
+ for value in (self.source_locator, self.declaration, self.namespace):
35
+ value.encode("utf-8")
36
+ except UnicodeError as exc:
37
+ raise RecordError("source provenance must be valid UTF-8 text") from exc
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class OrderedRecord:
42
+ id: str
43
+ occurrence_id: str
44
+ source_position: int
45
+ body: str
46
+ raw_text: str
47
+ prefix: str
48
+ marker: Optional[int]
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class OrderedRecords:
53
+ records: Tuple[OrderedRecord, ...]
54
+ order: SourceOrder
55
+ record_format: str
56
+ _original_json: str
57
+
58
+ def roundtrip(self):
59
+ """Original JSON values, including original prefixes and all metadata."""
60
+ return json.loads(self._original_json)
61
+
62
+ def diagnostics(self):
63
+ markers = [r.marker for r in self.records]
64
+ present = bool(markers) and all(m is not None for m in markers)
65
+ return dict(feature="declared_order_records_v1", experimental=True,
66
+ source_order=asdict(self.order), record_format=self.record_format,
67
+ input_records_sha256=hashlib.sha256(self._original_json.encode("utf-8")).hexdigest(),
68
+ record_ids=[r.id for r in self.records],
69
+ occurrence_ids=[r.occurrence_id for r in self.records],
70
+ marker_values=markers,
71
+ contiguous_markers=bool(present and all(b == a + 1 for a, b in zip(markers, markers[1:]))),
72
+ source_order_preserved=True, marker_is_revision_clock=False,
73
+ supersession_eligible=False, deleted_record_ids=[], policy="keep_all")
74
+
75
+
76
+ _PREFIX = {"step": re.compile(r"^(\[step (\d+)\] )(.*)$", re.DOTALL),
77
+ "integer_dot": re.compile(r"^((\d+)\. )(.*)$", re.DOTALL)}
78
+
79
+
80
+ def normalize_ordered_records(records, order, *, record_format="content"):
81
+ """Preserve each declared record as one body, with reversible prefix parsing.
82
+
83
+ `content` accepts content/text mappings or strings. `step` and `integer_dot`
84
+ accept the same explicit record sequence with a producer-declared prefix.
85
+ `serial` reads the shared leading-serial lexer without detector thresholds;
86
+ its marker is diagnostic and the declared array remains the source order.
87
+ A single plain-text blob supplies neither trustworthy boundaries nor order.
88
+ The source hash is a caller-supplied provenance assertion; this function
89
+ also records a canonical hash of the received JSON records.
90
+ """
91
+ if isinstance(order, dict):
92
+ try:
93
+ order = SourceOrder(**order)
94
+ except TypeError as exc:
95
+ raise RecordError("invalid source order fields") from exc
96
+ if not isinstance(order, SourceOrder):
97
+ raise RecordError("explicit SourceOrder provenance is required")
98
+ if record_format not in ("content", "step", "integer_dot", "serial"):
99
+ raise RecordError("record_format must be content, step, integer_dot or serial")
100
+ if not isinstance(records, (list, tuple)) or not records:
101
+ raise RecordError("explicit nonempty record array required; plain text has no declared boundaries")
102
+ try:
103
+ original = json.dumps(records, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
104
+ original.encode("utf-8")
105
+ except (TypeError, ValueError, UnicodeError) as exc:
106
+ raise RecordError("declared records must be valid UTF-8 JSON values") from exc
107
+ out, seen = [], set()
108
+ for i, row in enumerate(copy.deepcopy(records)):
109
+ occurrence = "%s@%s#record%d" % (order.namespace, order.source_sha256, i)
110
+ if isinstance(row, dict):
111
+ raw = row.get("text", row.get("content"))
112
+ if "text" in row and "content" in row and row["text"] != row["content"]:
113
+ raise RecordError("text and content disagree in record %d" % i)
114
+ rid = row.get("id", occurrence)
115
+ else:
116
+ raw, rid = row, occurrence
117
+ if not isinstance(raw, str):
118
+ raise RecordError("record %d needs a content/text string" % i)
119
+ if not isinstance(rid, (str, int)) or isinstance(rid, bool) or not str(rid).strip():
120
+ raise RecordError("record %d has an invalid ID" % i)
121
+ rid = str(rid)
122
+ if rid in seen:
123
+ raise RecordError("duplicate record ID %r" % rid)
124
+ seen.add(rid)
125
+ prefix, body, marker = "", raw, None
126
+ if record_format == "serial":
127
+ m = LEAD.match(raw)
128
+ if m is None or m.end(2) > 32:
129
+ raise RecordError("record %d does not match declared serial prefix" % i)
130
+ try:
131
+ marker = int(m.group(2))
132
+ except ValueError as exc:
133
+ raise RecordError("record %d has an unsupported serial integer" % i) from exc
134
+ prefix, body = raw[:m.end()], raw[m.end():]
135
+ elif record_format != "content":
136
+ m = _PREFIX[record_format].fullmatch(raw)
137
+ if m is None:
138
+ raise RecordError("record %d does not match declared %s prefix" % (i, record_format))
139
+ try:
140
+ marker = int(m.group(2))
141
+ except ValueError as exc:
142
+ raise RecordError("record %d has an unsupported serial integer" % i) from exc
143
+ prefix, body = m.group(1), m.group(3)
144
+ if not body.strip():
145
+ raise RecordError("record %d has an empty body" % i)
146
+ if prefix + body != raw:
147
+ raise AssertionError("record prefix roundtrip failed")
148
+ out.append(OrderedRecord(rid, occurrence, i, body, raw, prefix, marker))
149
+ return OrderedRecords(tuple(out), order, record_format, original)
150
+
151
+
152
+ def build_declared(envelope, *, models=None, embed=True):
153
+ """Build the optional record-preserving view from one JSON envelope."""
154
+ if not isinstance(envelope, dict) or set(envelope) - {"records", "order", "format"}:
155
+ raise RecordError("declared envelope has records, order and optional format only")
156
+ try:
157
+ records, order = envelope["records"], envelope["order"]
158
+ except KeyError as exc:
159
+ raise RecordError("declared envelope needs records and order") from exc
160
+ from .memory import build
161
+ return build(records, models=models, embed=embed, declared_order=order,
162
+ record_format=envelope.get("format", "content"))
163
+
164
+
165
+ def build_ordered_memory(records, order, *, record_format, plan, models, embed):
166
+ from . import bpe
167
+ from .cut import Cut
168
+ from .memory import Memory, resolve_plan
169
+ from .records import Record
170
+ normalized = normalize_ordered_records(records, order, record_format=record_format)
171
+ bodies = tuple(r.body for r in normalized.records)
172
+ text, spans, pos = "\n\n".join(bodies), [], 0
173
+ for body in bodies:
174
+ spans.append((pos, pos + len(body)))
175
+ pos += len(body) + 2
176
+ p = resolve_plan(text, plan)
177
+ if p.mode != "native":
178
+ raise RecordError("declared-order records are a native opt-in mode; exact replay is unchanged")
179
+ trace = ("Caller declares record boundaries and source-array/append order; no date sorting.",
180
+ "One body per unit, one parent per occurrence; all versions kept without a revision contract.",
181
+ "The base plan supplies U=%d; declared record granularity replaces automatic token packing." % p.U)
182
+ p = p.with_(branch="record", atom=None, chunk=None, mscale=0, policy="flrs", render="r4", supersede="off",
183
+ compat=None, source="declared-order:record-boundaries-v1",
184
+ trace=trace)
185
+ diagnostics = normalized.diagnostics()
186
+ c = Cut(bodies, bodies, tuple("u:" + r.occurrence_id for r in normalized.records),
187
+ tuple(range(len(bodies))), tuple(spans),
188
+ tuple(len(bpe.o200k().encode(b, disallowed_special=())) for b in bodies), None, diagnostics)
189
+ recs = [Record(text=r.body, id=r.id, meta={"occurrence_id": r.occurrence_id,
190
+ "source_position": str(r.source_position)}) for r in normalized.records]
191
+ mem = Memory(plan=p, text=text, cut=c, records=recs, record_spans=spans, models=models,
192
+ ordered_records=normalized)
193
+ if mem.index.structural or mem.index.newest_row:
194
+ raise AssertionError("source order unexpectedly enabled revision pruning")
195
+ if embed:
196
+ mem.embed(models)
197
+ return mem
muster/stats.py ADDED
@@ -0,0 +1,108 @@
1
+ """The two statistics that decide a store's representation, computed from its text with no model.
2
+
3
+ serial fraction the share of non-empty lines (first 20,000) that open with "<integer>. <body>"
4
+ record share the token-weighted share of blank-line records of at most the reference chunk
5
+
6
+ `store_stats` is scripts/derive.py:69-79 (one store, CRLF normalized, special-token strings read as text).
7
+ `collection_stats` is harness/derive_config.py:43-64 (the first rows of a benchmark cell, pooled), which the paper's
8
+ per-cell decisions used.
9
+
10
+ `lead_serial` is not in the paper: it finds a log that numbers its records in its own style ("Step 3: ..."),
11
+ and only `derive` reads it, after the serial fraction has failed.
12
+ """
13
+ import re
14
+ import statistics
15
+ from dataclasses import dataclass
16
+
17
+ from . import bpe
18
+ from .records import normalize_newlines
19
+
20
+ NUM = re.compile(r"^\s*\d+\.\s+\S")
21
+ MAX_LINES = 20000
22
+ # a serial under the log's own spelling: an optional bracket and word, the integer, an optional closer, then the body
23
+ # ("Step 3: ", "#3 ", "(3) ", "[3] ", "3. "). Thresholds set after the paper (2026-11) on public benchmark logs.
24
+ LEAD = re.compile(r"^(\s*[\[(#]?\s*(?:[A-Za-z]+\s*)?)(\d{1,8})([\]).:]?\s+)(?=\S)")
25
+ LEAD_TAU, LEAD_LINE_TOKENS = 0.95, 20
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class Stats:
30
+ serial: float
31
+ record_share: float
32
+ tokens: int # store: tokens in blank-line records; collection: mean whole-row tokens
33
+ lines_used: int
34
+ rows: int = 1
35
+ scope: str = "store"
36
+
37
+ def to_dict(self):
38
+ return dict(serial=self.serial, record_share=self.record_share, tokens=self.tokens,
39
+ lines_used=self.lines_used, rows=self.rows, scope=self.scope)
40
+
41
+
42
+ def _serial(text):
43
+ lines = [ln for ln in text.split("\n") if ln.strip()][:MAX_LINES]
44
+ return (sum(1 for ln in lines if NUM.match(ln)) / float(len(lines)) if lines else 0.0), len(lines)
45
+
46
+
47
+ def lead_match(s):
48
+ """LEAD at the start of `s` when its integer ends within the first 32 characters, else None."""
49
+ m = LEAD.match(s)
50
+ return m if m and m.end(2) <= 32 else None
51
+
52
+
53
+ def lead_serial(text):
54
+ """(share of lines after the first, among the first 20,000 non-empty lines, whose leading serial is the previous
55
+ line's plus one, median cl100k tokens per line). The median is counted only when the share reaches LEAD_TAU, else
56
+ it is None. A serial must end within a line's first 32 characters."""
57
+ lines = [ln for ln in normalize_newlines(text).split("\n") if ln.strip()][:MAX_LINES]
58
+ if not lines:
59
+ return 0.0, None
60
+ prev, step = None, 0
61
+ for ln in lines:
62
+ m = lead_match(ln)
63
+ v = int(m.group(2)) if m else None
64
+ if v is not None and prev is not None and v == prev + 1:
65
+ step += 1
66
+ prev = v
67
+ share = step / float(max(len(lines) - 1, 1))
68
+ if share < LEAD_TAU:
69
+ return share, None
70
+ enc = bpe.cl100k()
71
+ return share, statistics.median([bpe.count(enc, ln) for ln in lines])
72
+
73
+
74
+ def store_stats(text: str, reference_chunk: int = 4096) -> Stats:
75
+ enc = bpe.cl100k()
76
+ text = normalize_newlines(text)
77
+ total = fit = 0
78
+ for rec in text.split("\n\n"):
79
+ if rec.strip():
80
+ t = bpe.count(enc, rec)
81
+ total += t
82
+ fit += t if t <= reference_chunk else 0
83
+ serial, n = _serial(text)
84
+ return Stats(serial, (fit / float(total) if total else 0.0), total, n)
85
+
86
+
87
+ def collection_stats(texts, reference_chunk: int, *, sample: int = 3, mode: str = "native") -> Stats:
88
+ enc = bpe.cl100k()
89
+ enc_count = (lambda s: len(enc.encode(s))) if mode == "exact" else (lambda s: bpe.count(enc, s))
90
+ tot = fit = 0
91
+ fracs, toks, used = [], [], 0
92
+ rows = list(texts)[:sample]
93
+ for txt in rows:
94
+ txt = str(txt)
95
+ toks.append(enc_count(txt))
96
+ for p in txt.split("\n\n"):
97
+ if not p.strip():
98
+ continue
99
+ t = enc_count(p)
100
+ tot += t
101
+ if t <= reference_chunk:
102
+ fit += t
103
+ lines = [ln for ln in txt.split("\n") if ln.strip()][:MAX_LINES]
104
+ used += len(lines)
105
+ if lines:
106
+ fracs.append(sum(1 for ln in lines if NUM.match(ln)) / float(len(lines)))
107
+ return Stats(sum(fracs) / len(fracs) if fracs else 0.0, fit / float(tot) if tot else 0.0,
108
+ (sum(toks) // len(toks)) if toks else 0, used, rows=len(rows), scope="collection")
muster/store.py ADDED
@@ -0,0 +1,186 @@
1
+ """Store: a growing memory in one SQLite file, for services, agents and robots.
2
+
3
+ store = muster.Store.open("agent.muster")
4
+ store.add([{"speaker": "user", "text": "I moved to Lisbon in May.", "time": "2026-05-02"}])
5
+ print(store.context("Where does the user live now?").text)
6
+
7
+ `add` only appends records (one transaction, no model call), so it is fast and safe to call from a hot path. The
8
+ first search after an add builds the new version: the grown store is cut, indexed and planned exactly as a fresh
9
+ build of all its records would be, and only keys never embedded before are sent to the encoder. Reranker scores and
10
+ vectors are cached by text, so they survive restarts and growth.
11
+ """
12
+ import json
13
+ import os
14
+ import sqlite3
15
+ import threading
16
+ import time
17
+ from dataclasses import MISSING, dataclass, fields
18
+ from pathlib import Path
19
+ from typing import Optional
20
+
21
+ from .cache import ScoreCache, VectorCache, model_key
22
+ from .errors import StoreLocked
23
+ from .memory import Memory, build
24
+ from .records import Record, as_record
25
+
26
+
27
+ @dataclass(frozen=True)
28
+ class AddResult:
29
+ version: int
30
+ duplicate: bool
31
+ n_records: int
32
+
33
+
34
+ def _rec_json(r: Record):
35
+ d = {}
36
+ for f in fields(Record):
37
+ v = getattr(r, f.name)
38
+ default = f.default_factory() if f.default_factory is not MISSING else f.default
39
+ if f.name == "text" or v != default:
40
+ d[f.name] = dict(v) if f.name == "meta" else (list(v) if isinstance(v, tuple) else v)
41
+ return json.dumps(d, ensure_ascii=False, sort_keys=True)
42
+
43
+
44
+ class Store:
45
+ def __init__(self, path, *, models=None, plan=None, template=None, readonly=False):
46
+ self.path, self.models, self.plan, self.template, self.readonly = os.fsdecode(path), models, plan, template, readonly
47
+ self._lock = threading.RLock()
48
+ # URI escaping preserves literal '?', '#' and '%' in application paths.
49
+ # SQLite enforces read-only access and refuses to create a missing file.
50
+ target = Path(self.path).absolute().as_uri() + "?mode=ro" if readonly else self.path
51
+ self.db = sqlite3.connect(target, uri=readonly, check_same_thread=False, isolation_level=None)
52
+ if not readonly:
53
+ self.db.execute("PRAGMA journal_mode=WAL")
54
+ self.db.execute("PRAGMA synchronous=FULL")
55
+ self.db.execute("CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT)")
56
+ self.db.execute("CREATE TABLE IF NOT EXISTS records(seq INTEGER PRIMARY KEY, request_id TEXT, record TEXT)")
57
+ self.db.execute("CREATE TABLE IF NOT EXISTS requests(request_id TEXT PRIMARY KEY, version INT)")
58
+ self._head: Optional[Memory] = None
59
+ self._head_version = -1
60
+ self._scores = None
61
+ self._vectors = None
62
+ if not readonly:
63
+ pid = self.db.execute("SELECT v FROM meta WHERE k='writer'").fetchone()
64
+ if pid and pid[0] != str(os.getpid()) and _alive(int(pid[0])):
65
+ raise StoreLocked("%s is open for writing by process %s" % (path, pid[0]))
66
+ self.db.execute("INSERT OR REPLACE INTO meta VALUES ('writer', ?)", (str(os.getpid()),))
67
+
68
+ @classmethod
69
+ def open(cls, path, **kw):
70
+ return cls(path, **kw)
71
+
72
+ # ------------------------------------------------------------- writing
73
+ def add(self, records, *, request_id: Optional[str] = None) -> AddResult:
74
+ """Append records. A repeated request_id is acknowledged once and appends nothing."""
75
+ if self.readonly:
76
+ raise PermissionError("store opened readonly")
77
+ if isinstance(records, (str, Record)) or hasattr(records, "keys"):
78
+ records = [records]
79
+ recs = [as_record(r) for r in records]
80
+ with self._lock:
81
+ cur = self.db.cursor()
82
+ cur.execute("BEGIN IMMEDIATE")
83
+ try:
84
+ if request_id is not None:
85
+ r = cur.execute("SELECT version FROM requests WHERE request_id=?", (request_id,)).fetchone()
86
+ if r is not None:
87
+ cur.execute("COMMIT")
88
+ return AddResult(r[0], True, 0)
89
+ cur.executemany("INSERT INTO records(request_id, record) VALUES (?, ?)",
90
+ [(request_id, _rec_json(x)) for x in recs])
91
+ v = cur.execute("SELECT COUNT(*) FROM records").fetchone()[0]
92
+ if request_id is not None:
93
+ cur.execute("INSERT INTO requests VALUES (?, ?)", (request_id, v))
94
+ cur.execute("COMMIT")
95
+ except Exception:
96
+ cur.execute("ROLLBACK")
97
+ raise
98
+ return AddResult(v, False, len(recs))
99
+
100
+ # ------------------------------------------------------------- reading
101
+ @property
102
+ def version(self):
103
+ return self.db.execute("SELECT COUNT(*) FROM records").fetchone()[0]
104
+
105
+ def records(self):
106
+ return [as_record(json.loads(r[0])) for r in self.db.execute("SELECT record FROM records ORDER BY seq")]
107
+
108
+ def _models(self):
109
+ if self.models is None:
110
+ from .models import Models
111
+ self.models = Models()
112
+ return self.models
113
+
114
+ def head(self) -> Memory:
115
+ """The current version as a Memory, rebuilt only when records were added since the last call."""
116
+ with self._lock:
117
+ v = self.version
118
+ if self._head is not None and self._head_version == v:
119
+ return self._head
120
+ kw = {"template": self.template} if self.template else {}
121
+ mem = build(self.records(), self.plan, models=self._models(), embed=False, **kw)
122
+ mem.version = v
123
+ if len(mem):
124
+ self._embed(mem)
125
+ self._head, self._head_version = mem, v
126
+ return mem
127
+
128
+ def _embed(self, mem):
129
+ import numpy as np
130
+ m = self._models()
131
+ if self._vectors is None:
132
+ if self.readonly and not self.db.execute(
133
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='vectors'").fetchone():
134
+ self.db.execute("CREATE TEMP TABLE vectors(model TEXT, ksha BLOB, dim INT, vec BLOB, "
135
+ "PRIMARY KEY (model, ksha))")
136
+ self._vectors = VectorCache(self.db, model_key(m.encoder) + "|%d" % mem.plan.encoder.max_length,
137
+ create=not self.readonly)
138
+ keys = list(mem.cut.keys)
139
+ have = self._vectors.get_many(keys)
140
+ todo = [k for k in dict.fromkeys(keys) if k not in have]
141
+ if todo:
142
+ vec = m.encoder.encode(todo, batch_size=mem.plan.encoder.batch_size, normalize_embeddings=True)
143
+ new = {k: np.asarray(v, dtype=np.float32).tobytes() for k, v in zip(todo, vec)}
144
+ if not self.readonly: # a reader keeps new vectors in memory and writes nothing
145
+ self.db.execute("BEGIN")
146
+ self._vectors.put_many(new)
147
+ self.db.execute("COMMIT")
148
+ have.update(new)
149
+ mem.vecs = np.stack([np.frombuffer(have[k], dtype=np.float32) for k in keys])
150
+
151
+ def score_cache(self):
152
+ if self._scores is None:
153
+ self._scores = ScoreCache(None if (self.readonly or self.path == ":memory:") else self.path + ".scores",
154
+ model=model_key(self._models().reranker))
155
+ return self._scores
156
+
157
+ def search(self, query, **kw):
158
+ mem = self.head()
159
+ return mem.search(query, scores=self.score_cache(), models=self._models(), **kw)
160
+
161
+ def context(self, query, **kw):
162
+ mem = self.head()
163
+ style = kw.pop("style", None)
164
+ return mem.render(mem.search(query, scores=self.score_cache(), models=self._models(), **kw), style=style)
165
+
166
+ def close(self):
167
+ if not self.readonly:
168
+ try:
169
+ self.db.execute("DELETE FROM meta WHERE k='writer' AND v=?", (str(os.getpid()),))
170
+ except sqlite3.Error:
171
+ pass
172
+ self.db.close()
173
+
174
+ def __enter__(self):
175
+ return self
176
+
177
+ def __exit__(self, *a):
178
+ self.close()
179
+
180
+
181
+ def _alive(pid):
182
+ try:
183
+ os.kill(pid, 0)
184
+ return True
185
+ except OSError:
186
+ return False