graph-mind-memory 0.1.0__py3-none-any.whl

development_memory.py

@@ -0,0 +1,450 @@
+"""Persistent, append-only development memory for Graph-MIND dogfooding.
+
+The event log is the source of truth. ``resume_project`` derives a bounded
+session-continuity view from events visible at an effective/knowledge cutoff.
+It deliberately reports un-attested history instead of inferring absence.
+"""
+from __future__ import annotations
+
+from contextlib import contextmanager
+from pathlib import Path, PurePosixPath
+import hashlib
+import json
+import re
+import sqlite3
+import uuid
+
+from temporal_state import timestamp
+
+
+CONTRACT_VERSION = "development-memory-v0.1"
+EVENT_TYPES = frozenset({
+    "OBJECTIVE_SET",
+    "TASK_STARTED",
+    "TASK_COMPLETED",
+    "TASK_BLOCKED",
+    "TASK_CANCELLED",
+    "FILE_CHANGED",
+    "DECISION_RECORDED",
+    "TEST_RECORDED",
+    "NEXT_ACTION_SET",
+    "NEXT_ACTION_COMPLETED",
+    "SESSION_SUMMARY",
+    "EXPERIMENT_STARTED",
+    "EXPERIMENT_STOPPED",
+})
+ACTORS = frozenset({"user", "assistant", "system", "tool"})
+TEST_STATUSES = frozenset({"PASSED", "FAILED", "SKIPPED", "STOPPED"})
+EXPERIMENT_STATUSES = frozenset({"PASSED", "FAILED", "STOPPED"})
+EVENT_KEYS = frozenset({
+    "event_id", "project_id", "session_id", "event_type", "effective_at",
+    "known_at", "actor", "subject_id", "payload", "provenance",
+    "supersedes_event_id",
+})
+REQUIRED_PAYLOAD_FIELDS = {
+    "OBJECTIVE_SET": ("objective",),
+    "TASK_STARTED": ("title",),
+    "TASK_COMPLETED": ("outcome",),
+    "TASK_BLOCKED": ("reason",),
+    "TASK_CANCELLED": ("reason",),
+    "FILE_CHANGED": ("path", "summary", "reason"),
+    "DECISION_RECORDED": ("decision", "reason"),
+    "TEST_RECORDED": ("name", "status", "summary"),
+    "NEXT_ACTION_SET": ("action",),
+    "NEXT_ACTION_COMPLETED": ("outcome",),
+    "SESSION_SUMMARY": ("summary",),
+    "EXPERIMENT_STARTED": ("title",),
+    "EXPERIMENT_STOPPED": ("status", "reason"),
+}
+
+
+def _canonical_json(value):
+    return json.dumps(value, ensure_ascii=False, sort_keys=True,
+                      separators=(",", ":"), allow_nan=False)
+
+
+def _fingerprint(value):
+    return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest()
+
+
+def _text(value, error):
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(error)
+    return value.strip()
+
+
+def _portable_path(value):
+    raw = _text(value, "FILE_PATH_REQUIRED").replace("\\", "/")
+    if re.match(r"^[A-Za-z]:", raw) or raw.startswith("/"):
+        raise ValueError("PROJECT_RELATIVE_FILE_PATH_REQUIRED")
+    path = PurePosixPath(raw)
+    if ".." in path.parts or not path.parts:
+        raise ValueError("PROJECT_RELATIVE_FILE_PATH_REQUIRED")
+    return str(path)
+
+
+class DevelopmentMemoryStore:
+    """SQLite event store and deterministic project-resume projection."""
+
+    def __init__(self, path=":memory:", *, clock="SOURCE_LOCAL"):
+        if clock not in {"UTC", "SOURCE_LOCAL"}:
+            raise ValueError("INVALID_STORE_CLOCK")
+        self.path = path if path == ":memory:" else Path(path)
+        if self.path != ":memory:":
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.db = sqlite3.connect(str(self.path))
+        self.closed = False
+        self.db.row_factory = sqlite3.Row
+        self.db.execute("PRAGMA foreign_keys=ON")
+        self.db.executescript("""
+        CREATE TABLE IF NOT EXISTS development_memory_meta(
+          key TEXT PRIMARY KEY, value TEXT NOT NULL);
+        CREATE TABLE IF NOT EXISTS development_events(
+          revision INTEGER PRIMARY KEY AUTOINCREMENT,
+          event_id TEXT NOT NULL UNIQUE,
+          project_id TEXT NOT NULL,
+          session_id TEXT NOT NULL,
+          event_type TEXT NOT NULL,
+          effective_at TEXT NOT NULL,
+          known_at TEXT NOT NULL,
+          actor TEXT NOT NULL,
+          subject_id TEXT NOT NULL,
+          payload_json TEXT NOT NULL,
+          provenance_json TEXT NOT NULL,
+          supersedes_event_id TEXT,
+          event_fingerprint TEXT NOT NULL,
+          FOREIGN KEY(supersedes_event_id) REFERENCES development_events(event_id));
+        CREATE INDEX IF NOT EXISTS development_project_clock
+          ON development_events(project_id,known_at,effective_at,revision);
+        CREATE INDEX IF NOT EXISTS development_project_type
+          ON development_events(project_id,event_type,subject_id,revision);
+        CREATE INDEX IF NOT EXISTS development_session
+          ON development_events(project_id,session_id,revision);
+        """)
+        with self.db:
+            self.db.execute("INSERT OR IGNORE INTO development_memory_meta VALUES('store_id',?)",
+                            (uuid.uuid4().hex,))
+            self.db.execute("INSERT OR IGNORE INTO development_memory_meta VALUES('clock',?)",
+                            (clock,))
+            self.db.execute("INSERT OR IGNORE INTO development_memory_meta VALUES('contract_version',?)",
+                            (CONTRACT_VERSION,))
+        self.store_id = self.db.execute(
+            "SELECT value FROM development_memory_meta WHERE key='store_id'").fetchone()[0]
+        stored_clock = self.db.execute(
+            "SELECT value FROM development_memory_meta WHERE key='clock'").fetchone()[0]
+        stored_contract = self.db.execute(
+            "SELECT value FROM development_memory_meta WHERE key='contract_version'").fetchone()[0]
+        if stored_clock != clock or stored_contract != CONTRACT_VERSION:
+            self.close()
+            raise ValueError("STORE_CLOCK_OR_CONTRACT_CHANGED")
+        self.clock = clock
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    def close(self):
+        if not self.closed:
+            self.closed = True
+            self.db.close()
+
+    def _time(self, value):
+        return timestamp(value, self.clock).isoformat(timespec="microseconds")
+
+    def _validate_event(self, event):
+        if not isinstance(event, dict):
+            raise ValueError("DEVELOPMENT_EVENT_OBJECT_REQUIRED")
+        extra = set(event) - EVENT_KEYS
+        if extra:
+            raise ValueError("UNSUPPORTED_DEVELOPMENT_EVENT_FIELDS")
+        normalized = dict(event)
+        for key in ("event_id", "project_id", "session_id", "event_type", "actor", "subject_id"):
+            normalized[key] = _text(event.get(key), key.upper() + "_REQUIRED")
+        if normalized["event_type"] not in EVENT_TYPES:
+            raise ValueError("UNSUPPORTED_DEVELOPMENT_EVENT_TYPE")
+        if normalized["actor"] not in ACTORS:
+            raise ValueError("UNSUPPORTED_DEVELOPMENT_EVENT_ACTOR")
+        effective = self._time(event.get("effective_at"))
+        known = self._time(event.get("known_at"))
+        if effective > known:
+            raise ValueError("DEVELOPMENT_EVENT_AFTER_RECORDING")
+        normalized["effective_at"], normalized["known_at"] = effective, known
+
+        payload = event.get("payload")
+        if not isinstance(payload, dict):
+            raise ValueError("DEVELOPMENT_EVENT_PAYLOAD_REQUIRED")
+        payload = dict(payload)
+        for field in REQUIRED_PAYLOAD_FIELDS[normalized["event_type"]]:
+            payload[field] = _text(payload.get(field), field.upper() + "_REQUIRED")
+        if normalized["event_type"] == "FILE_CHANGED":
+            payload["path"] = _portable_path(payload["path"])
+        if normalized["event_type"] == "TEST_RECORDED" and payload["status"] not in TEST_STATUSES:
+            raise ValueError("INVALID_TEST_STATUS")
+        if (normalized["event_type"] == "EXPERIMENT_STOPPED"
+                and payload["status"] not in EXPERIMENT_STATUSES):
+            raise ValueError("INVALID_EXPERIMENT_STATUS")
+        normalized["payload"] = payload
+
+        provenance = event.get("provenance")
+        if not isinstance(provenance, dict):
+            raise ValueError("DEVELOPMENT_EVENT_PROVENANCE_REQUIRED")
+        provenance = dict(provenance)
+        for key in ("source_type", "source_ref"):
+            provenance[key] = _text(provenance.get(key), key.upper() + "_REQUIRED")
+        digest = provenance.get("content_sha256")
+        if digest is not None and (not isinstance(digest, str)
+                                   or re.fullmatch(r"[0-9a-f]{64}", digest) is None):
+            raise ValueError("INVALID_PROVENANCE_SHA256")
+        normalized["provenance"] = provenance
+
+        supersedes = event.get("supersedes_event_id")
+        if supersedes is not None:
+            supersedes = _text(supersedes, "INVALID_SUPERSEDES_EVENT_ID")
+            if supersedes == normalized["event_id"]:
+                raise ValueError("EVENT_CANNOT_SUPERSEDE_ITSELF")
+        normalized["supersedes_event_id"] = supersedes
+        return normalized
+
+    def _insert(self, event):
+        normalized = self._validate_event(event)
+        fingerprint = _fingerprint(normalized)
+        existing = self.db.execute(
+            "SELECT revision,event_fingerprint FROM development_events WHERE event_id=?",
+            (normalized["event_id"],)).fetchone()
+        if existing:
+            if existing["event_fingerprint"] != fingerprint:
+                raise ValueError("DEVELOPMENT_EVENT_ID_COLLISION")
+            return {"event_id": normalized["event_id"], "revision": existing["revision"],
+                    "disposition": "ALREADY_RECORDED"}
+
+        head = self.db.execute(
+            "SELECT known_at FROM development_events ORDER BY revision DESC LIMIT 1").fetchone()
+        if head and normalized["known_at"] < head["known_at"]:
+            raise ValueError("BACKDATED_DEVELOPMENT_KNOWLEDGE_MUTATION")
+
+        required_predecessor = {
+            "TASK_COMPLETED": "TASK_STARTED",
+            "TASK_BLOCKED": "TASK_STARTED",
+            "TASK_CANCELLED": "TASK_STARTED",
+            "NEXT_ACTION_COMPLETED": "NEXT_ACTION_SET",
+            "EXPERIMENT_STOPPED": "EXPERIMENT_STARTED",
+        }.get(normalized["event_type"])
+        if required_predecessor:
+            predecessor = self.db.execute("""SELECT 1 FROM development_events
+              WHERE project_id=? AND subject_id=? AND event_type=? LIMIT 1""",
+              (normalized["project_id"], normalized["subject_id"], required_predecessor)).fetchone()
+            if not predecessor:
+                raise ValueError(required_predecessor + "_NOT_FOUND")
+        if normalized["supersedes_event_id"]:
+            target = self.db.execute(
+                "SELECT project_id,event_type,subject_id FROM development_events WHERE event_id=?",
+                (normalized["supersedes_event_id"],)).fetchone()
+            if not target:
+                raise ValueError("SUPERSEDED_EVENT_NOT_FOUND")
+            if target["project_id"] != normalized["project_id"]:
+                raise ValueError("CROSS_PROJECT_SUPERSESSION_FORBIDDEN")
+            if (target["event_type"] != normalized["event_type"]
+                    or target["subject_id"] != normalized["subject_id"]):
+                raise ValueError("SUPERSESSION_TYPE_OR_SUBJECT_MISMATCH")
+        cursor = self.db.execute("""INSERT INTO development_events(
+          event_id,project_id,session_id,event_type,effective_at,known_at,actor,
+          subject_id,payload_json,provenance_json,supersedes_event_id,event_fingerprint)
+          VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""", (
+            normalized["event_id"], normalized["project_id"], normalized["session_id"],
+            normalized["event_type"], normalized["effective_at"], normalized["known_at"],
+            normalized["actor"], normalized["subject_id"],
+            _canonical_json(normalized["payload"]), _canonical_json(normalized["provenance"]),
+            normalized["supersedes_event_id"], fingerprint))
+        return {"event_id": normalized["event_id"], "revision": cursor.lastrowid,
+                "disposition": "RECORDED"}
+
+    def record_event(self, event):
+        with self.db:
+            return self._insert(event)
+
+    def record_events(self, events):
+        if not isinstance(events, list) or not events:
+            raise ValueError("NONEMPTY_DEVELOPMENT_EVENT_LIST_REQUIRED")
+        try:
+            self.db.execute("BEGIN IMMEDIATE")
+            results = [self._insert(event) for event in events]
+            self.db.commit()
+            return results
+        except BaseException:
+            self.db.rollback()
+            raise
+
+    @staticmethod
+    def _event(row):
+        return {"revision": row["revision"], "event_id": row["event_id"],
+                "project_id": row["project_id"], "session_id": row["session_id"],
+                "event_type": row["event_type"], "effective_at": row["effective_at"],
+                "known_at": row["known_at"], "actor": row["actor"],
+                "subject_id": row["subject_id"], "payload": json.loads(row["payload_json"]),
+                "provenance": json.loads(row["provenance_json"]),
+                "supersedes_event_id": row["supersedes_event_id"]}
+
+    def events(self, project_id, *, as_of, knowledge_cutoff=None):
+        project_id = _text(project_id, "PROJECT_ID_REQUIRED")
+        effective = self._time(as_of)
+        cutoff = self._time(knowledge_cutoff or as_of)
+        rows = self.db.execute("""SELECT * FROM development_events
+          WHERE project_id=? AND effective_at<=? AND known_at<=?
+          ORDER BY effective_at,known_at,revision""", (project_id, effective, cutoff)).fetchall()
+        events = [self._event(row) for row in rows]
+        superseded = {e["supersedes_event_id"] for e in events if e["supersedes_event_id"]}
+        return [e for e in events if e["event_id"] not in superseded]
+
+    def resume_project(self, project_id, *, as_of, knowledge_cutoff=None,
+                       recent_file_limit=10, recent_test_limit=10):
+        if type(recent_file_limit) is not int or recent_file_limit < 1:
+            raise ValueError("INVALID_RECENT_FILE_LIMIT")
+        if type(recent_test_limit) is not int or recent_test_limit < 1:
+            raise ValueError("INVALID_RECENT_TEST_LIMIT")
+        selected = self.events(project_id, as_of=as_of,
+                               knowledge_cutoff=knowledge_cutoff)
+        cutoff = self._time(knowledge_cutoff or as_of)
+        effective = self._time(as_of)
+        if not selected:
+            return {"status": "UNKNOWN", "reason": "PROJECT_MEMORY_NOT_FOUND",
+                    "project_id": project_id, "as_of": effective,
+                    "knowledge_cutoff": cutoff,
+                    "scope_completeness": "UNATTESTED", "evidence": []}
+
+        objective = None
+        tasks, files, decisions, next_actions, experiments = {}, {}, {}, {}, {}
+        tests, sessions = [], []
+        event_by_id = {e["event_id"]: e for e in selected}
+
+        for event in selected:
+            kind, subject, payload = event["event_type"], event["subject_id"], event["payload"]
+            if kind == "OBJECTIVE_SET":
+                objective = {"value": payload["objective"],
+                             "evidence_event_ids": [event["event_id"]]}
+            elif kind == "TASK_STARTED":
+                old = tasks.get(subject, {})
+                tasks[subject] = {"task_id": subject, "title": payload["title"],
+                                  "status": "OPEN", "detail": None,
+                                  "last_revision": event["revision"],
+                                  "evidence_event_ids": old.get("evidence_event_ids", []) + [event["event_id"]]}
+            elif kind in {"TASK_COMPLETED", "TASK_BLOCKED", "TASK_CANCELLED"}:
+                old = tasks.get(subject, {"task_id": subject, "title": subject,
+                                          "evidence_event_ids": []})
+                status = {"TASK_COMPLETED": "COMPLETED", "TASK_BLOCKED": "BLOCKED",
+                          "TASK_CANCELLED": "CANCELLED"}[kind]
+                detail = payload.get("outcome") or payload.get("reason")
+                tasks[subject] = {**old, "status": status, "detail": detail,
+                                  "last_revision": event["revision"],
+                                  "evidence_event_ids": old["evidence_event_ids"] + [event["event_id"]]}
+            elif kind == "FILE_CHANGED":
+                files[payload["path"]] = {"path": payload["path"],
+                    "summary": payload["summary"], "reason": payload["reason"],
+                    "last_revision": event["revision"],
+                    "evidence_event_ids": [event["event_id"]]}
+            elif kind == "DECISION_RECORDED":
+                decisions[subject] = {"decision_id": subject, "decision": payload["decision"],
+                    "reason": payload["reason"], "last_revision": event["revision"],
+                    "evidence_event_ids": [event["event_id"]]}
+            elif kind == "TEST_RECORDED":
+                tests.append({"test_id": subject, "name": payload["name"],
+                    "status": payload["status"], "summary": payload["summary"],
+                    "last_revision": event["revision"],
+                    "evidence_event_ids": [event["event_id"]]})
+            elif kind == "NEXT_ACTION_SET":
+                old = next_actions.get(subject, {})
+                next_actions[subject] = {"action_id": subject, "action": payload["action"],
+                    "status": "OPEN", "last_revision": event["revision"],
+                    "evidence_event_ids": old.get("evidence_event_ids", []) + [event["event_id"]]}
+            elif kind == "NEXT_ACTION_COMPLETED":
+                old = next_actions.get(subject, {"action_id": subject, "action": subject,
+                                                 "evidence_event_ids": []})
+                next_actions[subject] = {**old, "status": "COMPLETED",
+                    "outcome": payload["outcome"], "last_revision": event["revision"],
+                    "evidence_event_ids": old["evidence_event_ids"] + [event["event_id"]]}
+            elif kind == "EXPERIMENT_STARTED":
+                old = experiments.get(subject, {})
+                experiments[subject] = {"experiment_id": subject, "title": payload["title"],
+                    "status": "RUNNING", "last_revision": event["revision"],
+                    "evidence_event_ids": old.get("evidence_event_ids", []) + [event["event_id"]]}
+            elif kind == "EXPERIMENT_STOPPED":
+                old = experiments.get(subject, {"experiment_id": subject, "title": subject,
+                                                "evidence_event_ids": []})
+                experiments[subject] = {**old, "status": payload["status"],
+                    "reason": payload["reason"], "last_revision": event["revision"],
+                    "evidence_event_ids": old["evidence_event_ids"] + [event["event_id"]]}
+            elif kind == "SESSION_SUMMARY":
+                sessions.append({"session_id": subject, "summary": payload["summary"],
+                    "last_revision": event["revision"],
+                    "evidence_event_ids": [event["event_id"]]})
+
+        open_tasks = sorted((v for v in tasks.values() if v["status"] in {"OPEN", "BLOCKED"}),
+                            key=lambda v: v["last_revision"], reverse=True)
+        current_task = open_tasks[0] if open_tasks else None
+        open_actions = sorted((v for v in next_actions.values() if v["status"] == "OPEN"),
+                              key=lambda v: v["last_revision"], reverse=True)
+        active_experiments = sorted((v for v in experiments.values() if v["status"] == "RUNNING"),
+                                    key=lambda v: v["last_revision"], reverse=True)
+        recent_files = sorted(files.values(), key=lambda v: v["last_revision"], reverse=True)[:recent_file_limit]
+        recent_tests = sorted(tests, key=lambda v: v["last_revision"], reverse=True)[:recent_test_limit]
+        decision_list = sorted(decisions.values(), key=lambda v: v["last_revision"], reverse=True)
+        last_session = max(sessions, key=lambda v: v["last_revision"]) if sessions else None
+
+        projected = [objective, current_task, *open_tasks, *open_actions, *active_experiments,
+                     *recent_files, *recent_tests, *decision_list, last_session]
+        evidence_ids = []
+        for item in projected:
+            if item:
+                evidence_ids.extend(item.get("evidence_event_ids", []))
+        evidence_ids = list(dict.fromkeys(evidence_ids))
+        evidence = [event_by_id[event_id] for event_id in evidence_ids if event_id in event_by_id]
+        evidence.sort(key=lambda e: e["revision"])
+
+        return {"status": "KNOWN", "reason": "PROJECT_MEMORY_RECONSTRUCTED",
+                "project_id": project_id, "as_of": effective, "knowledge_cutoff": cutoff,
+                "scope_completeness": "UNATTESTED",
+                "warnings": ["PROJECT_HISTORY_COMPLETENESS_NOT_ATTESTED"],
+                "objective": objective, "current_task": current_task,
+                "open_tasks": open_tasks, "next_actions": open_actions,
+                "active_experiments": active_experiments, "recent_files": recent_files,
+                "recent_tests": recent_tests, "decisions": decision_list,
+                "last_session": last_session, "evidence_event_ids": evidence_ids,
+                "evidence": evidence}
+
+    def explain_decision(self, project_id, decision_id, *, as_of,
+                         knowledge_cutoff=None):
+        project_id = _text(project_id, "PROJECT_ID_REQUIRED")
+        decision_id = _text(decision_id, "DECISION_ID_REQUIRED")
+        effective = self._time(as_of)
+        cutoff = self._time(knowledge_cutoff or as_of)
+        rows = self.db.execute("""SELECT * FROM development_events
+          WHERE project_id=? AND subject_id=? AND event_type='DECISION_RECORDED'
+          AND effective_at<=? AND known_at<=?
+          ORDER BY effective_at,known_at,revision""",
+          (project_id, decision_id, effective, cutoff)).fetchall()
+        history = [self._event(row) for row in rows]
+        if not history:
+            return {"status": "UNKNOWN", "reason": "DECISION_NOT_FOUND",
+                    "project_id": project_id, "decision_id": decision_id,
+                    "as_of": effective, "knowledge_cutoff": cutoff,
+                    "scope_completeness": "UNATTESTED", "history": []}
+        superseded = {event["supersedes_event_id"] for event in history
+                      if event["supersedes_event_id"]}
+        active = [event for event in history if event["event_id"] not in superseded]
+        if len(active) != 1:
+            return {"status": "UNKNOWN", "reason": "CONFLICTING_ACTIVE_DECISIONS",
+                    "project_id": project_id, "decision_id": decision_id,
+                    "as_of": effective, "knowledge_cutoff": cutoff,
+                    "scope_completeness": "UNATTESTED",
+                    "evidence_event_ids": [event["event_id"] for event in active],
+                    "history": history}
+        current = active[0]
+        return {"status": "KNOWN", "reason": "DECISION_RECONSTRUCTED",
+                "project_id": project_id, "decision_id": decision_id,
+                "as_of": effective, "knowledge_cutoff": cutoff,
+                "scope_completeness": "UNATTESTED",
+                "decision": current["payload"]["decision"],
+                "decision_reason": current["payload"]["reason"],
+                "evidence_event_ids": [current["event_id"]],
+                "provenance": current["provenance"], "history": history}

development_paths.py

@@ -0,0 +1,40 @@
+"""Stable user-owned paths shared by every Graph-MIND client."""
+from pathlib import Path
+import hashlib
+import os
+
+
+def graph_mind_home():
+    configured = os.environ.get("GRAPH_MIND_HOME")
+    return Path(configured).expanduser() if configured else Path.home() / ".graph-mind"
+
+
+def default_development_db():
+    return graph_mind_home() / "development.sqlite"
+
+
+def default_capture_state_dir(project_id):
+    suffix = hashlib.sha256(project_id.encode("utf-8")).hexdigest()[:16]
+    return graph_mind_home() / "capture" / suffix
+
+
+def default_coding_state_dir(project_id):
+    suffix = hashlib.sha256(project_id.encode("utf-8")).hexdigest()[:16]
+    return graph_mind_home() / "coding" / suffix
+
+
+def use_wal(db):
+    """Put a store in WAL mode, waiting for the other clients if need be. Changing the journal
+    mode needs the database to itself, and SQLite does not wait for that the way it waits for a
+    write: three clients opening a new store at once failed with "database is locked" (macOS CI).
+    """
+    import sqlite3
+    import time
+    for attempt in range(100):
+        try:
+            db.execute("PRAGMA journal_mode=WAL")
+            return
+        except sqlite3.OperationalError as error:
+            if "locked" not in str(error) or attempt == 99:
+                raise
+            time.sleep(0.1)

embedding_warmup.py

@@ -0,0 +1,171 @@
+"""Embed memories when they are STORED, not when they are asked about.
+
+semantic_recall embeds each candidate's text at query time. The vectors are cached, so the cost is
+paid once — but it is paid by whoever asks the first question after a batch of memories arrives, and
+on CPU that is seconds to minutes. Writing is the right moment: it happens once per memory, in the
+background of a conversation the user is already having.
+
+The text embedded here is exactly what the recall path will hash (the engram's FTS text, first 500
+characters), so a warmed memory is a cache hit rather than a near miss.
+
+    python embedding_warmup.py [db_path]      # warm a store, or self-check with no arguments
+"""
+from __future__ import annotations
+
+from pathlib import Path
+import sqlite3
+import sys
+
+EXCERPT = 500          # semantic_recall embeds text_excerpt = text[:500]
+BATCH = 128
+
+
+def pending_texts(db: sqlite3.Connection, embedder, limit: int) -> list[str]:
+    """Engram excerpts that have no vector yet, newest first."""
+    rows = db.execute(
+        "SELECT f.text FROM associative_engrams_fts f "
+        "JOIN associative_engrams e ON e.engram_id = f.engram_id "
+        "ORDER BY e.revision DESC LIMIT ?", (limit,)).fetchall()
+    texts, seen = [], set()
+    for (text,) in rows:
+        excerpt = (text or "")[:EXCERPT].strip()[:2000]
+        if excerpt and excerpt not in seen and embedder._key(excerpt) not in embedder.cache:
+            seen.add(excerpt)
+            texts.append(excerpt)
+    return texts
+
+
+def warm(db_path, *, limit: int = 500, embedder=None) -> int:
+    """Embed everything unembedded in the newest `limit` engrams. Returns how many were added."""
+    from semantic_recall import default_embedder
+
+    embedder = embedder or default_embedder(db_path)
+    db = sqlite3.connect(db_path)
+    try:
+        db.execute("SELECT 1 FROM associative_engrams LIMIT 1")
+    except sqlite3.OperationalError:
+        db.close()
+        return 0                       # no associative index yet: nothing to warm
+    try:
+        texts = pending_texts(db, embedder, limit)
+        for start in range(0, len(texts), BATCH):
+            embedder.embed(texts[start:start + BATCH])
+        if texts and hasattr(embedder, "flush"):
+            embedder.flush()
+        return len(texts)
+    finally:
+        db.close()
+
+
+PIECE = 500           # conversation turns are searched by meaning in 500-character pieces
+
+
+def turn_pieces(text: str) -> list[tuple[int, str]]:
+    """(offset, piece) for one turn, exactly as the recall path cuts and searches it."""
+    return [(start, text[start:start + PIECE]) for start in range(0, len(text), PIECE)
+            if text[start:start + PIECE].strip()]
+
+
+def warm_turns(db_path, *, limit: int = 512, embedder=None) -> int:
+    """Embed up to `limit` unembedded pieces of conversation turns, the user's first, newest first.
+
+    The capture service calls this after every pass, so the turns it has just stored are
+    searchable by meaning before anyone asks about them. Without it the first question after a
+    busy day met thousands of unembedded pieces, and the server, which never makes a question
+    wait for that, answered it from word search alone (79.2% against 86.0% on LongMemEval)."""
+    from semantic_recall import default_embedder
+    embedder = embedder or default_embedder(db_path)
+    if hasattr(embedder.cache, "refresh"):
+        embedder.cache.refresh()                 # what the memory servers embedded meanwhile
+    db = sqlite3.connect(db_path, timeout=30)
+    try:
+        # the user's turns first (every search uses them), then the assistant's (searched when
+        # the question is about what it said: left to the server, a fresh server spent minutes
+        # embedding all of them on the first such question, and every query waited meanwhile)
+        rows = db.execute("SELECT content_redacted FROM conversation_turns "
+                          "WHERE role IN ('user','assistant') "
+                          "ORDER BY role='user' DESC, happened_at DESC")
+        texts, seen = [], set()
+        for (content,) in rows:
+            for _, piece in turn_pieces(content or ""):
+                key = piece.strip()[:2000]
+                if key not in seen and embedder._key(key) not in embedder.cache:
+                    seen.add(key)
+                    texts.append(piece)
+            if len(texts) >= limit:
+                break
+    except sqlite3.OperationalError as error:
+        if "no such table" not in str(error):
+            raise
+        return 0                                 # no conversations captured yet
+    finally:
+        db.close()
+    texts = texts[:limit]
+    for start in range(0, len(texts), BATCH):
+        embedder.embed(texts[start:start + BATCH])
+    if texts and hasattr(embedder, "flush"):
+        embedder.flush()
+    return len(texts)
+
+
+def _self_check():
+    import tempfile
+    from associative_memory import AssociativeMemoryIndex
+    from local_brain import LocalBrainStore
+
+    path = Path(tempfile.mkdtemp()) / "brain.sqlite"
+    with LocalBrainStore(path) as store:
+        for i, title in enumerate(["JWT expiry cut to 1h", "Bought an air fryer"], start=1):
+            store.remember({"memory_id": f"m{i}", "scope": "global", "memory_type": "fact",
+                            "title": title, "content": title + " — details follow",
+                            "effective_at": "2026-04-02T09:00:00",
+                            "known_at": "2026-04-02T09:00:00", "actor": "assistant",
+                            "tags": [], "entities": ["auth module"],
+                            "provenance": {"source_type": "test", "source_ref": "unit"},
+                            "supersedes_memory_id": None})
+    with AssociativeMemoryIndex(path) as index:
+        index.sync_sources()
+
+    class FakeEmbedder:
+        def __init__(self):
+            self.cache, self.calls = {}, 0
+
+        def _key(self, text):
+            return "k:" + text[:40]
+
+        def embed(self, texts):
+            self.calls += 1
+            for text in texts:
+                self.cache.setdefault(self._key(text), [0.1, 0.2])
+            return [self.cache[self._key(t)] for t in texts]
+
+    fake = FakeEmbedder()
+    first = warm(path, embedder=fake)
+    assert first >= 2, first                      # both memories embedded
+    assert warm(path, embedder=fake) == 0         # nothing left to do
+    calls = fake.calls
+    assert warm(path, embedder=fake) == 0 and fake.calls == calls, "warm must not re-embed"
+
+    db = sqlite3.connect(path)
+    texts = pending_texts(db, fake, 500)
+    db.close()
+    assert texts == [], texts
+    with sqlite3.connect(path) as db:
+        db.execute("CREATE TABLE conversation_turns(content_redacted TEXT, role TEXT, happened_at TEXT)")
+        db.executemany("INSERT INTO conversation_turns VALUES(?,?,?)", [
+            ("".join(f"{i:04d}" for i in range(300)), "user", "2026-01-01"), ("answer " * 300, "assistant", "2026-01-02"),
+            ("newest question", "user", "2026-01-03")])
+    turns = FakeEmbedder()
+    assert warm_turns(path, embedder=turns, limit=2) == 2          # newest first, capped
+    assert turns._key("newest question") in turns.cache
+    assert not any("answer" in key for key in turns.cache), "the user's turns come first"
+    assert warm_turns(path, embedder=turns) == 7                    # 2 user + 5 assistant pieces
+    assert warm_turns(path, embedder=turns) == 0
+    print(f"embedding_warmup self-check ok ({first} vectors warmed, second pass free)")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        print("warmed", warm(sys.argv[1]), "vectors")
+    else:
+        _self_check()