abstractnote 0.1.0__py3-none-any.whl

abstractnote/store/queue.py

@@ -0,0 +1,159 @@
+"""The processing queue (docs/cognition.md, "note-drain and the queue").
+
+Rows live in the note index ``queue`` table. Order: priority desc, occurred_at
+asc, size asc. A lease lasts 15 minutes; an expired lease is taken over by the
+next drain and counts as a failed attempt. A failure schedules a retry after
+1 minute, then 10 minutes; the third failure marks the note ``failed``
+(``RETRY_BACKOFF_S[2]``, one hour, is the delay before the attempt that follows
+a third failure when ``MAX_FAILURES`` is raised). Leasing runs in one
+``BEGIN IMMEDIATE`` transaction, so two drains never lease the same note.
+"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any, Dict, List, Optional
+
+from .. import ids
+
+LEASE_S = 15 * 60
+RETRY_BACKOFF_S = (60, 10 * 60, 60 * 60)
+MAX_FAILURES = 3
+PRIORITY_BULK, PRIORITY_NORMAL, PRIORITY_HIGH, PRIORITY_URGENT = 0, 1, 2, 3
+STOPPED_ERROR = "Processing stopped before it finished."
+
+
+def _plus(ts: str, seconds: int) -> str:
+    return (ids.parse_time(ts) + timedelta(seconds=seconds)).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+class Queue:
+    def __init__(self, store: Any) -> None:
+        self.store = store
+
+    # ------------------------------------------------------------------ writes
+
+    def enqueue(self, conn: Any, note_id: str, priority: int, occurred_at: str, size: int, *,
+                now: Optional[str] = None) -> None:
+        """Add a note (inside the caller's transaction). Re-enqueueing keeps the existing row."""
+        conn.execute(
+            "INSERT OR IGNORE INTO queue(note_id, priority, occurred_at, size, enqueued_at, state)"
+            " VALUES (?,?,?,?,?,'queued')", (note_id, int(priority), occurred_at, int(size), now or ids.utc_now()))
+        conn.execute("UPDATE notes SET status='queued' WHERE note_id=? AND status='received'", (note_id,))
+
+    def set_priority(self, conn: Any, note_id: str, priority: int) -> None:
+        conn.execute("UPDATE queue SET priority=? WHERE note_id=?", (int(priority), note_id))
+
+    def lease(self, worker_id: str, max_items: int = 1, *, now: Optional[str] = None) -> Dict[str, Any]:
+        worker = str(worker_id or "").strip()
+        if not worker:
+            raise ValueError("A worker id is needed to take notes from the queue.")
+        n = max(1, min(100, int(max_items or 1)))
+        now = now or ids.utc_now()
+        items: List[Dict[str, Any]] = []
+        with self.store.tx() as c:
+            # Expired leases: the drain stopped; that counts as a failure.
+            for r in c.execute("SELECT note_id, failures FROM queue WHERE state='leased' AND lease_until<=?",
+                               (now,)).fetchall():
+                self._fail(c, r["note_id"], int(r["failures"]) + 1, STOPPED_ERROR, now)
+            rows = c.execute(
+                "SELECT note_id, priority, attempts FROM queue WHERE state='queued' AND"
+                " (next_attempt_at IS NULL OR next_attempt_at<=?)"
+                " ORDER BY priority DESC, occurred_at ASC, size ASC, note_id ASC LIMIT ?", (now, n)).fetchall()
+            until = _plus(now, LEASE_S)
+            for r in rows:
+                cur = c.execute("UPDATE queue SET state='leased', leased_by=?, lease_until=?, attempts=attempts+1"
+                                " WHERE note_id=? AND state='queued'", (worker, until, r["note_id"]))
+                if cur.rowcount != 1:
+                    continue
+                c.execute("UPDATE notes SET status='processing' WHERE note_id=? AND archived_at IS NULL",
+                          (r["note_id"],))
+                items.append({"note_id": r["note_id"], "priority": r["priority"], "attempts": r["attempts"] + 1,
+                              "lease_until": until})
+        return {"items": items}
+
+    def _fail(self, c: Any, note_id: str, failures: int, error: str, now: str) -> Dict[str, Any]:
+        if failures >= MAX_FAILURES:
+            c.execute("UPDATE queue SET state='failed', failures=?, last_error=?, failed_at=?, leased_by=NULL,"
+                      " lease_until=NULL, next_attempt_at=NULL WHERE note_id=?", (failures, error, now, note_id))
+            c.execute("UPDATE notes SET status='failed' WHERE note_id=? AND archived_at IS NULL", (note_id,))
+            return {"note_id": note_id, "status": "failed"}
+        nxt = _plus(now, RETRY_BACKOFF_S[min(failures - 1, len(RETRY_BACKOFF_S) - 1)])
+        c.execute("UPDATE queue SET state='queued', failures=?, last_error=?, leased_by=NULL, lease_until=NULL,"
+                  " next_attempt_at=? WHERE note_id=?", (failures, error, nxt, note_id))
+        c.execute("UPDATE notes SET status='queued' WHERE note_id=? AND archived_at IS NULL", (note_id,))
+        return {"note_id": note_id, "status": "queued", "next_attempt_at": nxt}
+
+    def ack(self, note_id: str, outcome: str, error: Optional[str] = None, revision_id: Optional[str] = None, *,
+            now: Optional[str] = None) -> Dict[str, Any]:
+        if outcome not in ("ok", "error"):
+            raise ValueError("The outcome must be ok or error.")
+        now = now or ids.utc_now()
+        with self.store.tx() as c:
+            row = c.execute("SELECT * FROM queue WHERE note_id=?", (note_id,)).fetchone()
+            if row is None:
+                raise ValueError("That note is not in the queue.")
+            if outcome == "ok":
+                if revision_id is not None and c.execute(
+                        "SELECT 1 FROM revisions WHERE revision_id=? AND note_id=? AND applied_index=1",
+                        (revision_id, note_id)).fetchone() is None:
+                    raise ValueError("That revision was not applied to this note.")
+                c.execute("UPDATE queue SET state='done', done_at=?, leased_by=NULL, lease_until=NULL,"
+                          " next_attempt_at=NULL WHERE note_id=?", (now, note_id))
+                c.execute("UPDATE notes SET status='processed', processed_at=COALESCE(processed_at, ?)"
+                          " WHERE note_id=? AND archived_at IS NULL", (now, note_id))
+                return {"note_id": note_id, "status": "processed"}
+            if row["state"] == "done":
+                return {"note_id": note_id, "status": "processed"}
+            return self._fail(c, note_id, int(row["failures"]) + 1, str(error or "Processing failed."), now)
+
+    def retry(self, note_id: str, *, now: Optional[str] = None) -> Dict[str, Any]:
+        """Re-queue a failed note (user action from the queue panel)."""
+        with self.store.tx() as c:
+            row = c.execute("SELECT state FROM queue WHERE note_id=?", (note_id,)).fetchone()
+            if row is None:
+                raise ValueError("That note is not in the queue.")
+            if row["state"] != "failed":
+                raise ValueError("Only a note that failed can be retried.")
+            c.execute("UPDATE queue SET state='queued', failures=0, next_attempt_at=NULL, failed_at=NULL"
+                      " WHERE note_id=?", (note_id,))
+            c.execute("UPDATE notes SET status='queued' WHERE note_id=? AND archived_at IS NULL", (note_id,))
+        return {"note_id": note_id, "status": "queued"}
+
+    # ------------------------------------------------------------------ reads
+
+    def attempt_for(self, note_id: str) -> int:
+        """The attempt number a revision made now would carry (the current lease, or the next one)."""
+        row = self.store.q1("SELECT state, attempts FROM queue WHERE note_id=?", (note_id,))
+        if row is None:
+            return 1
+        return int(row["attempts"]) if row["state"] == "leased" and row["attempts"] else int(row["attempts"]) + 1
+
+    def item(self, note_id: str) -> Optional[Dict[str, Any]]:
+        row = self.store.q1("SELECT * FROM queue WHERE note_id=?", (note_id,))
+        return dict(row) if row else None
+
+    def stats(self) -> Dict[str, Any]:
+        rows = self.store.q("SELECT state, priority, COUNT(*) AS n FROM queue GROUP BY state, priority")
+        out: Dict[str, Any] = {"queued": 0, "processing": 0, "failed": 0, "done": 0,
+                               "by_priority": {str(p): 0 for p in (3, 2, 1, 0)}}
+        for r in rows:
+            key = {"queued": "queued", "leased": "processing", "failed": "failed", "done": "done"}[r["state"]]
+            out[key] += r["n"]
+            if r["state"] in ("queued", "leased"):
+                out["by_priority"][str(r["priority"])] = out["by_priority"].get(str(r["priority"]), 0) + r["n"]
+        oldest = self.store.q1("SELECT MIN(enqueued_at) FROM queue WHERE state IN ('queued','leased')")
+        out["oldest_enqueued_at"] = oldest[0] if oldest else None
+        return out
+
+    def summary(self, limit: int = 50) -> Dict[str, Any]:
+        """Queue state and failed notes with their errors (``GET /note/queue``)."""
+        failed = self.store.q(
+            "SELECT q.note_id, q.last_error, q.failed_at, q.failures, n.title, n.occurred_at FROM queue q"
+            " JOIN notes n ON n.note_id=q.note_id WHERE q.state='failed' ORDER BY q.failed_at DESC LIMIT ?",
+            (int(limit),))
+        waiting = self.store.q(
+            "SELECT q.note_id, q.priority, q.state, q.attempts, q.next_attempt_at, n.title, n.occurred_at FROM queue q"
+            " JOIN notes n ON n.note_id=q.note_id WHERE q.state IN ('queued','leased')"
+            " ORDER BY q.priority DESC, q.occurred_at ASC, q.size ASC LIMIT ?", (int(limit),))
+        return {"stats": self.stats(), "failed": [dict(r) for r in failed], "waiting": [dict(r) for r in waiting]}

abstractnote/store/schema.py

@@ -0,0 +1,348 @@
+"""SQLite schema of the note index (``note.sqlite3``).
+
+Truth tables (kept by a rebuild): ``meta``, ``notes``, ``attachments``,
+``note_links``, ``queue``, ``revisions``, ``vectors`` (the embedding cache) and
+``context_packets``. Everything else is a projection of notes plus revisions
+and is recreated by ``Notebook.rebuild``.
+
+Vectors are float32 blobs; cosine search runs in Python over the filtered rows.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+SCHEMA_VERSION = 1
+
+TRUTH_TABLES = ("meta", "notes", "attachments", "note_links", "queue", "revisions", "vectors", "context_packets")
+
+DDL = """
+CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
+
+CREATE TABLE IF NOT EXISTS notes (
+  note_id TEXT PRIMARY KEY,
+  kind TEXT,
+  title TEXT,
+  text TEXT NOT NULL DEFAULT '',
+  text_norm TEXT NOT NULL DEFAULT '',
+  text_hash TEXT,
+  one_line TEXT,
+  summary TEXT,
+  occurred_at TEXT NOT NULL,
+  received_at TEXT NOT NULL,
+  processed_at TEXT,
+  source_json TEXT NOT NULL,
+  client TEXT,
+  location_json TEXT,
+  language TEXT,
+  filing_topics_json TEXT NOT NULL DEFAULT '[]',
+  priority INTEGER NOT NULL DEFAULT 1,
+  priority_explicit INTEGER NOT NULL DEFAULT 0,
+  status TEXT NOT NULL DEFAULT 'received',
+  duplicate_of TEXT,
+  session_id TEXT,
+  run_id TEXT,
+  archived_at TEXT,
+  idempotency_key TEXT,
+  size INTEGER NOT NULL DEFAULT 0,
+  vector_key TEXT,
+  insight_kind TEXT,
+  evidence_note_ids_json TEXT,
+  entity_ids_json TEXT,
+  acked_at TEXT,
+  about_json TEXT,
+  seq INTEGER
+);
+CREATE INDEX IF NOT EXISTS idx_notes_occurred ON notes(occurred_at);
+CREATE INDEX IF NOT EXISTS idx_notes_received ON notes(received_at);
+CREATE INDEX IF NOT EXISTS idx_notes_text_hash ON notes(text_hash);
+CREATE INDEX IF NOT EXISTS idx_notes_kind ON notes(kind);
+
+CREATE TABLE IF NOT EXISTS attachments (
+  attachment_id TEXT PRIMARY KEY,
+  note_id TEXT NOT NULL,
+  sha256 TEXT NOT NULL,
+  size INTEGER NOT NULL,
+  mime TEXT NOT NULL,
+  filename TEXT,
+  artifact_ref_json TEXT,
+  blob_path TEXT NOT NULL,
+  extracted_by TEXT NOT NULL DEFAULT 'none',
+  extracted_chars INTEGER NOT NULL DEFAULT 0,
+  created_at TEXT NOT NULL,
+  position INTEGER NOT NULL DEFAULT 0
+);
+CREATE INDEX IF NOT EXISTS idx_attachments_note ON attachments(note_id);
+CREATE INDEX IF NOT EXISTS idx_attachments_sha ON attachments(sha256);
+
+CREATE TABLE IF NOT EXISTS note_links (
+  src_note_id TEXT NOT NULL,
+  dst_note_id TEXT NOT NULL,
+  predicate TEXT NOT NULL,
+  score REAL,
+  matched_by TEXT,
+  detail_json TEXT,
+  created_at TEXT NOT NULL,
+  PRIMARY KEY (src_note_id, dst_note_id, predicate)
+);
+CREATE INDEX IF NOT EXISTS idx_note_links_dst ON note_links(dst_note_id);
+
+CREATE TABLE IF NOT EXISTS queue (
+  note_id TEXT PRIMARY KEY,
+  priority INTEGER NOT NULL,
+  occurred_at TEXT NOT NULL,
+  size INTEGER NOT NULL DEFAULT 0,
+  enqueued_at TEXT NOT NULL,
+  state TEXT NOT NULL DEFAULT 'queued',
+  leased_by TEXT,
+  lease_until TEXT,
+  attempts INTEGER NOT NULL DEFAULT 0,
+  failures INTEGER NOT NULL DEFAULT 0,
+  next_attempt_at TEXT,
+  last_error TEXT,
+  failed_at TEXT,
+  done_at TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_queue_order ON queue(state, priority DESC, occurred_at ASC, size ASC);
+
+CREATE TABLE IF NOT EXISTS revisions (
+  seq INTEGER PRIMARY KEY AUTOINCREMENT,
+  revision_id TEXT NOT NULL UNIQUE,
+  note_id TEXT,
+  attempt INTEGER,
+  action TEXT,
+  target TEXT,
+  at TEXT NOT NULL,
+  recorded_at TEXT NOT NULL,
+  run_id TEXT,
+  ops_json TEXT NOT NULL,
+  summary TEXT NOT NULL DEFAULT '',
+  source_json TEXT,
+  warnings_json TEXT NOT NULL DEFAULT '[]',
+  result_json TEXT,
+  applied_index INTEGER NOT NULL DEFAULT 0,
+  applied_kg INTEGER NOT NULL DEFAULT 0,
+  abandoned INTEGER NOT NULL DEFAULT 0,
+  error TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_revisions_note ON revisions(note_id);
+CREATE INDEX IF NOT EXISTS idx_revisions_at ON revisions(at);
+CREATE INDEX IF NOT EXISTS idx_revisions_recorded ON revisions(recorded_at);
+
+CREATE TABLE IF NOT EXISTS vectors (
+  vector_key TEXT PRIMARY KEY,
+  model_id TEXT NOT NULL,
+  dimension INTEGER NOT NULL,
+  vector BLOB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS context_packets (
+  note_id TEXT PRIMARY KEY,
+  created_at TEXT NOT NULL,
+  packet_json TEXT NOT NULL
+);
+
+-- ---------------------------------------------------------------- projections
+
+CREATE TABLE IF NOT EXISTS entities (
+  entity_id TEXT PRIMARY KEY,
+  kind TEXT NOT NULL,
+  name TEXT NOT NULL,
+  summary TEXT,
+  created_at TEXT NOT NULL,
+  created_revision_id TEXT NOT NULL,
+  recorded_at TEXT NOT NULL,
+  first_seen TEXT,
+  last_seen TEXT,
+  note_count INTEGER NOT NULL DEFAULT 0,
+  merged_into TEXT,
+  merged_at TEXT,
+  vector_key TEXT,
+  revision_count INTEGER NOT NULL DEFAULT 0
+);
+CREATE INDEX IF NOT EXISTS idx_entities_created ON entities(created_at);
+CREATE INDEX IF NOT EXISTS idx_entities_last_seen ON entities(last_seen);
+
+CREATE TABLE IF NOT EXISTS entity_aliases (
+  entity_id TEXT NOT NULL,
+  alias TEXT NOT NULL,
+  alias_fold TEXT NOT NULL,
+  added_at TEXT NOT NULL,
+  revision_id TEXT NOT NULL,
+  PRIMARY KEY (entity_id, alias_fold)
+);
+CREATE INDEX IF NOT EXISTS idx_alias_fold ON entity_aliases(alias_fold);
+
+CREATE TABLE IF NOT EXISTS entity_names (
+  entity_id TEXT NOT NULL,
+  name TEXT NOT NULL,
+  at TEXT NOT NULL,
+  revision_id TEXT NOT NULL,
+  PRIMARY KEY (entity_id, revision_id)
+);
+
+CREATE TABLE IF NOT EXISTS entity_summaries (
+  entity_id TEXT NOT NULL,
+  revision_id TEXT NOT NULL,
+  at TEXT NOT NULL,
+  note_id TEXT,
+  summary TEXT NOT NULL,
+  previous TEXT,
+  PRIMARY KEY (entity_id, revision_id)
+);
+
+CREATE TABLE IF NOT EXISTS entity_notes (
+  entity_id TEXT NOT NULL,
+  note_id TEXT NOT NULL,
+  at TEXT NOT NULL,
+  revision_id TEXT NOT NULL,
+  PRIMARY KEY (entity_id, note_id)
+);
+CREATE INDEX IF NOT EXISTS idx_entity_notes_note ON entity_notes(note_id);
+
+CREATE TABLE IF NOT EXISTS merges (
+  from_id TEXT NOT NULL,
+  into_id TEXT NOT NULL,
+  at TEXT NOT NULL,
+  revision_id TEXT NOT NULL,
+  reason TEXT,
+  undone_at TEXT,
+  undone_revision_id TEXT,
+  assertion_id TEXT NOT NULL,
+  PRIMARY KEY (from_id, revision_id)
+);
+
+CREATE TABLE IF NOT EXISTS claims (
+  claim_id TEXT PRIMARY KEY,
+  subject_id TEXT NOT NULL,
+  predicate TEXT NOT NULL,
+  object_id TEXT,
+  object_literal TEXT,
+  certainty TEXT NOT NULL,
+  stance TEXT NOT NULL,
+  observed_at TEXT NOT NULL,
+  valid_from TEXT,
+  valid_until TEXT,
+  status TEXT NOT NULL DEFAULT 'active',
+  superseded_by TEXT,
+  closed_reason TEXT,
+  closed_revision_id TEXT,
+  evidence TEXT,
+  note_id TEXT,
+  revision_id TEXT NOT NULL,
+  recorded_at TEXT NOT NULL,
+  assertion_id TEXT NOT NULL,
+  vector_key TEXT,
+  disputed_with_json TEXT NOT NULL DEFAULT '[]',
+  reinforced_count INTEGER NOT NULL DEFAULT 0,
+  seq INTEGER
+);
+CREATE INDEX IF NOT EXISTS idx_claims_subject ON claims(subject_id, predicate);
+CREATE INDEX IF NOT EXISTS idx_claims_object ON claims(object_id);
+CREATE INDEX IF NOT EXISTS idx_claims_observed ON claims(observed_at);
+CREATE INDEX IF NOT EXISTS idx_claims_note ON claims(note_id);
+
+CREATE TABLE IF NOT EXISTS claim_reinforcements (
+  claim_id TEXT NOT NULL,
+  note_id TEXT,
+  revision_id TEXT NOT NULL,
+  at TEXT NOT NULL,
+  PRIMARY KEY (claim_id, revision_id)
+);
+
+CREATE TABLE IF NOT EXISTS relations (
+  subject_id TEXT NOT NULL,
+  object_id TEXT NOT NULL,
+  predicate TEXT NOT NULL,
+  first_seen TEXT NOT NULL,
+  last_seen TEXT NOT NULL,
+  count INTEGER NOT NULL,
+  active_claim_ids_json TEXT NOT NULL DEFAULT '[]',
+  PRIMARY KEY (subject_id, object_id, predicate)
+);
+CREATE INDEX IF NOT EXISTS idx_relations_object ON relations(object_id);
+
+CREATE TABLE IF NOT EXISTS topics (
+  topic TEXT PRIMARY KEY,
+  first_seen TEXT,
+  last_seen TEXT,
+  note_count INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS note_topics (
+  note_id TEXT NOT NULL,
+  topic TEXT NOT NULL,
+  assigned_by TEXT NOT NULL,
+  revision_id TEXT,
+  at TEXT NOT NULL,
+  PRIMARY KEY (note_id, topic)
+);
+CREATE INDEX IF NOT EXISTS idx_note_topics_topic ON note_topics(topic);
+
+CREATE TABLE IF NOT EXISTS tasks (
+  task_id TEXT PRIMARY KEY,
+  note_id TEXT,
+  text TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'open',
+  due TEXT,
+  opened_at TEXT NOT NULL,
+  opened_revision_id TEXT NOT NULL,
+  closed_at TEXT,
+  closed_by_note_id TEXT,
+  closed_revision_id TEXT,
+  recorded_closed_at TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status);
+
+CREATE TABLE IF NOT EXISTS then_now_cache (
+  key TEXT PRIMARY KEY,
+  revision_count INTEGER NOT NULL,
+  payload_json TEXT NOT NULL
+);
+"""
+
+FTS_DDL = """
+CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(note_id UNINDEXED, title, body, summary, attachments);
+CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(entity_id UNINDEXED, name, aliases, summary);
+CREATE VIRTUAL TABLE IF NOT EXISTS claims_fts USING fts5(claim_id UNINDEXED, text);
+"""
+
+PROJECTION_TABLES = (
+    "entities", "entity_aliases", "entity_names", "entity_summaries", "entity_notes", "merges", "claims",
+    "claim_reinforcements", "relations", "topics", "note_topics", "tasks", "then_now_cache",
+    "entities_fts", "claims_fts",
+)
+
+
+def connect(path: str) -> sqlite3.Connection:
+    conn = sqlite3.connect(path, check_same_thread=False, isolation_level=None, timeout=30.0)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=30000")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute("PRAGMA foreign_keys=OFF")
+    return conn
+
+
+# Columns added after the first notebooks were created (in-place upgrade on open).
+_ADDED_COLUMNS = {"notes": [("about_json", "TEXT")]}
+
+
+def ensure_schema(conn: sqlite3.Connection) -> None:
+    conn.executescript(DDL)
+    for table, cols in _ADDED_COLUMNS.items():
+        have = {r[1] for r in conn.execute(f"PRAGMA table_info({table})")}
+        for name, decl in cols:
+            if name not in have:
+                conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {decl}")
+    try:
+        conn.executescript(FTS_DDL)
+    except sqlite3.OperationalError as exc:  # pragma: no cover - every supported Python ships FTS5
+        raise RuntimeError("AbstractNote needs SQLite with FTS5; this Python's sqlite3 lacks it.") from exc
+    row = conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone()
+    if row is None:
+        conn.execute("INSERT INTO meta(key, value) VALUES ('schema_version', ?)", (str(SCHEMA_VERSION),))
+    elif int(row[0]) > SCHEMA_VERSION:
+        raise RuntimeError(
+            f"This notebook was written by a newer AbstractNote (schema {row[0]}); upgrade abstractnote to open it."
+        )

abstractnote/store/vector_index.py

@@ -0,0 +1,88 @@
+"""In-memory cosine index over note vectors, refreshed incrementally from the note index.
+
+Captures, quick search and the context packet all need "nearest notes"; reading
+every vector from SQLite for each call made capture time grow with the
+notebook. The index loads rows above the last ``notes.seq`` it saw (note
+vectors never change except through a rebuild, which resets the index), so
+other processes writing to the same notebook are picked up on the next call.
+"""
+
+from __future__ import annotations
+
+import threading
+from typing import Any, Iterable, List, Optional, Sequence, Set, Tuple
+
+from .embeddings import _np, cosine, unpack
+
+
+class NoteVectorIndex:
+    def __init__(self, store: Any) -> None:
+        self.store = store
+        self._lock = threading.RLock()
+        self.reset()
+
+    def reset(self) -> None:
+        with self._lock:
+            self._last_seq = 0
+            self._ids: List[str] = []
+            self._insight: List[bool] = []
+            self._rows: List[bytes] = []
+            self._mat = None
+            self._dirty = False
+
+    def refresh(self) -> None:
+        rows = self.store.q(
+            "SELECT n.seq, n.note_id, n.insight_kind, v.vector FROM notes n JOIN vectors v ON v.vector_key=n.vector_key"
+            " WHERE n.seq>? ORDER BY n.seq", (self._last_seq,))
+        if not rows:
+            return
+        with self._lock:
+            for r in rows:
+                self._ids.append(r["note_id"])
+                self._insight.append(r["insight_kind"] is not None)
+                self._rows.append(r["vector"])
+                self._last_seq = max(self._last_seq, int(r["seq"]))
+            self._dirty = True
+
+    def nearest(self, query: Sequence[float], k: int, min_score: Optional[float] = None, *,
+                allowed: Optional[Set[str]] = None, exclude: Iterable[str] = (), notes_only: bool = False
+                ) -> List[Tuple[str, float]]:
+        self.refresh()
+        excluded = set(exclude)
+        with self._lock:
+            if not self._ids:
+                return []
+            if _np is not None:
+                if self._dirty or self._mat is None:
+                    have = 0 if self._mat is None else self._mat.shape[0]
+                    new = _np.frombuffer(b"".join(self._rows[have:]), dtype="<f4").reshape(len(self._rows) - have, -1)
+                    norms = _np.linalg.norm(new, axis=1)
+                    norms[norms == 0] = 1.0
+                    if self._mat is None:
+                        self._mat, self._norms = new, norms
+                    else:
+                        self._mat = _np.vstack([self._mat, new])
+                        self._norms = _np.concatenate([self._norms, norms])
+                    self._dirty = False
+                q = _np.asarray(query, dtype=_np.float32)
+                if q.shape[0] != self._mat.shape[1]:
+                    raise ValueError("A stored vector has a different dimension from the query; rebuild with --reembed.")
+                qn = float(_np.linalg.norm(q)) or 1.0
+                scores = (self._mat @ q) / (self._norms * qn)
+                order = _np.argsort(-scores)
+                pairs = ((int(i), float(scores[i])) for i in order)
+            else:  # pragma: no cover - numpy ships with the framework
+                scored = [(i, cosine(query, unpack(b))) for i, b in enumerate(self._rows)]
+                scored.sort(key=lambda t: -t[1])
+                pairs = iter(scored)
+            out: List[Tuple[str, float]] = []
+            for i, s in pairs:
+                if min_score is not None and s < min_score:
+                    break
+                nid = self._ids[i]
+                if nid in excluded or (notes_only and self._insight[i]) or (allowed is not None and nid not in allowed):
+                    continue
+                out.append((nid, s))
+                if len(out) >= k:
+                    break
+            return out