abstractnote 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. abstractnote/__init__.py +18 -0
  2. abstractnote/cli.py +240 -0
  3. abstractnote/flows/__init__.py +2 -0
  4. abstractnote/flows/build.py +970 -0
  5. abstractnote/flows/bundle.py +62 -0
  6. abstractnote/flows/contract.py +88 -0
  7. abstractnote/flows/graph.py +540 -0
  8. abstractnote/flows/json/note-capture.json +2280 -0
  9. abstractnote/flows/json/note-drain.json +2012 -0
  10. abstractnote/flows/json/note-mine.json +2392 -0
  11. abstractnote/flows/json/note-process.json +1853 -0
  12. abstractnote/flows/json/note-recall.json +1876 -0
  13. abstractnote/flows/json/note-reflect.json +2057 -0
  14. abstractnote/flows/prompts_fallback.py +119 -0
  15. abstractnote/flows/runner.py +320 -0
  16. abstractnote/flows/schemas_fallback.py +175 -0
  17. abstractnote/gateway/__init__.py +24 -0
  18. abstractnote/gateway/appspec.py +10 -0
  19. abstractnote/gateway/router.py +571 -0
  20. abstractnote/gateway/tools.py +74 -0
  21. abstractnote/ids.py +177 -0
  22. abstractnote/importers/__init__.py +142 -0
  23. abstractnote/importers/__main__.py +3 -0
  24. abstractnote/importers/agora.py +203 -0
  25. abstractnote/importers/claude_sessions.py +415 -0
  26. abstractnote/importers/cli.py +113 -0
  27. abstractnote/importers/files.py +72 -0
  28. abstractnote/importers/markdown.py +165 -0
  29. abstractnote/ingest/__init__.py +7 -0
  30. abstractnote/ingest/blobs.py +47 -0
  31. abstractnote/ingest/capture.py +265 -0
  32. abstractnote/ingest/extract.py +118 -0
  33. abstractnote/ingest/normalise.py +95 -0
  34. abstractnote/model.py +229 -0
  35. abstractnote/process/__init__.py +5 -0
  36. abstractnote/process/context.py +218 -0
  37. abstractnote/process/merge.py +60 -0
  38. abstractnote/process/prompts/__init__.py +39 -0
  39. abstractnote/process/prompts/extract.md +30 -0
  40. abstractnote/process/prompts/link.md +32 -0
  41. abstractnote/process/prompts/mine_dossier.md +13 -0
  42. abstractnote/process/prompts/mine_findings.md +13 -0
  43. abstractnote/process/prompts/mine_plan.md +13 -0
  44. abstractnote/process/prompts/quick_filing.md +23 -0
  45. abstractnote/process/prompts/reflect.md +10 -0
  46. abstractnote/process/prompts/synthesize.md +17 -0
  47. abstractnote/process/resolve.py +96 -0
  48. abstractnote/process/revise.py +410 -0
  49. abstractnote/process/schemas.py +377 -0
  50. abstractnote/process/vocabulary.py +68 -0
  51. abstractnote/recall/__init__.py +1 -0
  52. abstractnote/recall/changes.py +191 -0
  53. abstractnote/recall/expand.py +379 -0
  54. abstractnote/recall/insights.py +62 -0
  55. abstractnote/recall/reflect.py +202 -0
  56. abstractnote/recall/review.py +191 -0
  57. abstractnote/recall/search.py +205 -0
  58. abstractnote/recall/shapes.py +52 -0
  59. abstractnote/recall/then_now.py +100 -0
  60. abstractnote/store/__init__.py +12 -0
  61. abstractnote/store/archive.py +166 -0
  62. abstractnote/store/embeddings.py +275 -0
  63. abstractnote/store/kg.py +219 -0
  64. abstractnote/store/notebook.py +829 -0
  65. abstractnote/store/notes.py +1190 -0
  66. abstractnote/store/queue.py +159 -0
  67. abstractnote/store/schema.py +348 -0
  68. abstractnote/store/vector_index.py +88 -0
  69. abstractnote/tools/__init__.py +215 -0
  70. abstractnote/tools/capture.py +121 -0
  71. abstractnote/tools/context.py +19 -0
  72. abstractnote/tools/process.py +22 -0
  73. abstractnote/tools/queue.py +23 -0
  74. abstractnote/tools/recall.py +64 -0
  75. abstractnote-0.1.0.dist-info/METADATA +145 -0
  76. abstractnote-0.1.0.dist-info/RECORD +80 -0
  77. abstractnote-0.1.0.dist-info/WHEEL +5 -0
  78. abstractnote-0.1.0.dist-info/entry_points.txt +2 -0
  79. abstractnote-0.1.0.dist-info/licenses/LICENSE +21 -0
  80. abstractnote-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,166 @@
1
+ """The archive as it stood on a date: the third time axis (docs/data-model.md "Time").
2
+
3
+ ``as_received = R`` answers "what did my archive hold on R?": only the notes received by R
4
+ (``notes.received_at <= R``) count, so material imported later is left out even when its own dates are
5
+ older. Everything derived from notes follows them: a revision belongs to the archive at R when the note
6
+ that caused it was received by R, and a user action (no note) when it was recorded by R. Entities exist
7
+ when a revision in the archive created them or a note in the archive mentions them; merges, names,
8
+ aliases, summaries and claim closures count only when their revision belongs; a claim closed by a later
9
+ revision is read as it was before that revision (replayed from the revisions that do belong).
10
+
11
+ ``at`` (history time) composes with it: the graph or card "as of T, from what the archive held on R".
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from typing import Any, Dict, List, Optional, Tuple
18
+
19
+ from .. import ids
20
+
21
+ FOREVER = "9999-12-31T23:59:59Z"
22
+
23
+ NOTES_SQL = "SELECT note_id FROM notes WHERE received_at<=?"
24
+ REVISIONS_SQL = ("SELECT r.revision_id FROM revisions r LEFT JOIN notes n ON n.note_id=r.note_id"
25
+ " WHERE r.applied_index=1 AND COALESCE(n.received_at, r.recorded_at)<=?")
26
+
27
+
28
+ class Archive:
29
+ """What the archive held at ``as_received``; reads ask it which rows belong."""
30
+
31
+ def __init__(self, store: Any, as_received: str) -> None:
32
+ self.store = store
33
+ self.at = ids.normalize_time(as_received, field="as_received")
34
+ self.revisions = {r[0] for r in store.q(REVISIONS_SQL, (self.at,))}
35
+ self._merges: Optional[Dict[str, List[Tuple[str, str, Optional[str]]]]] = None
36
+ self._claim_states: Dict[str, Dict[str, Any]] = {}
37
+
38
+ # ---------------------------------------------------------------- SQL filters
39
+ def notes_in(self, col: str) -> Tuple[str, List[Any]]:
40
+ """``col IN (notes received by R)``."""
41
+ return f"{col} IN ({NOTES_SQL})", [self.at]
42
+
43
+ def revisions_in(self, col: str) -> Tuple[str, List[Any]]:
44
+ """``col IN (revisions that belong to the archive at R)``."""
45
+ return f"{col} IN ({REVISIONS_SQL})", [self.at]
46
+
47
+ def has(self, revision_id: Optional[str]) -> bool:
48
+ return bool(revision_id) and revision_id in self.revisions
49
+
50
+ # ---------------------------------------------------------------- merges
51
+ def merges(self) -> Dict[str, List[Tuple[str, str, Optional[str]]]]:
52
+ """``from_id -> [(at, into_id, undone_at or None)]`` for the merges in the archive, newest first; an undo
53
+ counts only when its own revision belongs."""
54
+ if self._merges is None:
55
+ out: Dict[str, List[Tuple[str, str, Optional[str]]]] = {}
56
+ for r in self.store.q("SELECT * FROM merges ORDER BY at DESC, revision_id DESC"):
57
+ if r["revision_id"] not in self.revisions:
58
+ continue
59
+ undone = r["undone_at"] if self.has(r["undone_revision_id"]) else None
60
+ out.setdefault(r["from_id"], []).append((r["at"], r["into_id"], undone))
61
+ self._merges = out
62
+ return self._merges
63
+
64
+ def canonical(self, entity_id: str, at: Optional[str] = None) -> str:
65
+ """The surviving entity as of history time ``at`` (default: any time), by the merges in the archive."""
66
+ ref = at or FOREVER
67
+ merges = self.merges()
68
+ seen = set()
69
+ cur = entity_id
70
+ while cur not in seen:
71
+ seen.add(cur)
72
+ nxt = None
73
+ for m_at, into, undone in merges.get(cur, []):
74
+ if m_at <= ref and (undone is None or undone > ref):
75
+ nxt = into
76
+ break
77
+ if nxt is None:
78
+ return cur
79
+ cur = nxt
80
+ return cur
81
+
82
+ def members(self, entity_id: str, at: Optional[str] = None) -> List[str]:
83
+ """The entity and every entry merged into it (transitively) in the archive as of ``at``."""
84
+ out = [entity_id]
85
+ for m in sorted(self.merges()):
86
+ if m != entity_id and self.canonical(m, at) == entity_id:
87
+ out.append(m)
88
+ return out
89
+
90
+ # ---------------------------------------------------------------- entities
91
+ def entity_stats(self) -> Dict[str, Dict[str, Any]]:
92
+ """``entity_id -> {first_seen, last_seen, notes}`` of each entity in the archive, from its own creation
93
+ (when the creating revision belongs) and its own notes in the archive (``notes`` is a set of note ids)."""
94
+ out: Dict[str, Dict[str, Any]] = {}
95
+ sql, params = self.revisions_in("created_revision_id")
96
+ for r in self.store.q(f"SELECT entity_id, created_at FROM entities WHERE {sql}", params):
97
+ out[r[0]] = {"first_seen": r[1], "last_seen": r[1], "notes": set()}
98
+ sql, params = self.notes_in("note_id")
99
+ for r in self.store.q(f"SELECT entity_id, note_id, at FROM entity_notes WHERE {sql}", params):
100
+ st = out.setdefault(r[0], {"first_seen": r[2], "last_seen": r[2], "notes": set()})
101
+ st["notes"].add(r[1])
102
+ st["first_seen"] = min(st["first_seen"], r[2])
103
+ st["last_seen"] = max(st["last_seen"], r[2])
104
+ return out
105
+
106
+ def holds_entity(self, entity_id: str) -> bool:
107
+ sql, params = self.revisions_in("created_revision_id")
108
+ if self.store.q1(f"SELECT 1 FROM entities WHERE entity_id=? AND {sql}", [entity_id, *params]):
109
+ return True
110
+ sql, params = self.notes_in("note_id")
111
+ return self.store.q1(f"SELECT 1 FROM entity_notes WHERE entity_id=? AND {sql}", [entity_id, *params]) is not None
112
+
113
+ def name(self, entity_id: str, at: Optional[str] = None) -> Optional[str]:
114
+ sql, params = self.revisions_in("revision_id")
115
+ r = self.store.q1(f"SELECT name FROM entity_names WHERE entity_id=? AND at<=? AND {sql} ORDER BY at DESC,"
116
+ " revision_id DESC LIMIT 1", [entity_id, at or FOREVER, *params])
117
+ return r[0] if r else None
118
+
119
+ def summary(self, entity_id: str, at: Optional[str] = None) -> Optional[str]:
120
+ sql, params = self.revisions_in("revision_id")
121
+ r = self.store.q1(f"SELECT summary FROM entity_summaries WHERE entity_id=? AND at<=? AND {sql} ORDER BY at"
122
+ " DESC, revision_id DESC LIMIT 1", [entity_id, at or FOREVER, *params])
123
+ return r[0] if r else None
124
+
125
+ # ---------------------------------------------------------------- claims
126
+ def claim_state(self, row: Any) -> Dict[str, Any]:
127
+ """``{status, valid_until, superseded_by}`` of a claim as the archive at R held it."""
128
+ cid = row["claim_id"]
129
+ if not row["closed_revision_id"] or row["closed_revision_id"] in self.revisions:
130
+ return {"status": row["status"], "valid_until": row["valid_until"], "superseded_by": row["superseded_by"]}
131
+ if cid in self._claim_states:
132
+ return self._claim_states[cid]
133
+ valid_until = None
134
+ added = self.store.revision_row(row["revision_id"])
135
+ for op in json.loads(added["ops_json"]) if added else []:
136
+ p = op.get("payload") or {}
137
+ if op.get("op") == "claim.add" and p.get("claim_id") == cid:
138
+ valid_until = p.get("valid_until")
139
+ state = {"status": "active", "valid_until": valid_until, "superseded_by": None}
140
+ for r in self.store.q("SELECT revision_id, at, ops_json FROM revisions WHERE applied_index=1 AND ops_json LIKE ?"
141
+ " ORDER BY seq", (f'%"{cid}"%',)):
142
+ if r["revision_id"] not in self.revisions:
143
+ continue
144
+ for op in json.loads(r["ops_json"]):
145
+ p = op.get("payload") or {}
146
+ if p.get("claim_id") != cid:
147
+ continue
148
+ if op.get("op") == "claim.supersede" and state["status"] == "active":
149
+ state = {"status": "superseded", "valid_until": state["valid_until"] or p.get("valid_until"),
150
+ "superseded_by": p.get("by_claim_id")}
151
+ elif op.get("op") == "claim.retract" and state["status"] != "retracted":
152
+ state = {"status": "retracted",
153
+ "valid_until": state["valid_until"] or p.get("valid_until") or r["at"],
154
+ "superseded_by": state["superseded_by"]}
155
+ self._claim_states[cid] = state
156
+ return state
157
+
158
+ def claim_dict(self, row: Any) -> Dict[str, Any]:
159
+ return {**self.store.claim_dict(row), **self.claim_state(row)}
160
+
161
+
162
+ def archive_for(store: Any, as_received: Optional[str]) -> Optional[Archive]:
163
+ """An :class:`Archive` for a non-empty ``as_received``, else None (the archive as it is now)."""
164
+ if as_received in (None, ""):
165
+ return None
166
+ return Archive(store, as_received)
@@ -0,0 +1,275 @@
1
+ """Embedders and the notebook's embedding cache.
2
+
3
+ One notebook = one embedding space. The pin ``{model_id, dimension}`` is kept
4
+ in the note index ``meta`` table and given to the AbstractMemory store at
5
+ creation. Every vector is computed once through :class:`VectorCache` (keyed by
6
+ the sha256 of the embedded text and the model id), stored in the note index,
7
+ and handed to the triple store, so a rebuild never re-embeds unless asked.
8
+
9
+ Adapters:
10
+
11
+ - :class:`HashingEmbedder` — deterministic feature hashing, offline; the
12
+ default for headless notebooks and the test suite.
13
+ - :func:`openai_compat_embedder` — any OpenAI-compatible ``/v1/embeddings``.
14
+ - :func:`gateway_embedder` — the gateway embedding route via
15
+ ``abstractmemory.AbstractGatewayTextEmbedder``.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import hashlib
21
+ import math
22
+ import re
23
+ import sqlite3
24
+ import struct
25
+ import threading
26
+ from typing import Any, Dict, Iterable, List, Optional, Protocol, Sequence, Tuple
27
+
28
+ try: # numpy is optional: faster cosine scans when present
29
+ import numpy as _np
30
+ except Exception: # pragma: no cover
31
+ _np = None
32
+
33
+ HASHING_MODEL_PREFIX = "abstractnote-hashing-v1"
34
+
35
+
36
+ class TextEmbedder(Protocol):
37
+ def embed_texts(self, texts: Sequence[str]) -> List[List[float]]: ...
38
+
39
+
40
+ _TOKEN_RE = re.compile(r"\w+", re.UNICODE)
41
+
42
+
43
+ class HashingEmbedder:
44
+ """Deterministic, offline embedder: signed feature hashing of words and
45
+ character trigrams, L2-normalised. Good enough for exact and near-exact
46
+ similarity (tests, headless use); not a semantic model."""
47
+
48
+ def __init__(self, dimension: int = 256) -> None:
49
+ if dimension < 8:
50
+ raise ValueError("The hashing embedder needs at least 8 dimensions.")
51
+ self.dimension = int(dimension)
52
+ self.model_id = f"{HASHING_MODEL_PREFIX}-{self.dimension}"
53
+
54
+ def _features(self, text: str) -> Iterable[str]:
55
+ words = _TOKEN_RE.findall(str(text or "").casefold())
56
+ for w in words:
57
+ yield "w:" + w
58
+ padded = f"#{w}#"
59
+ for i in range(len(padded) - 2):
60
+ yield "c:" + padded[i : i + 3]
61
+
62
+ def embed_texts(self, texts: Sequence[str]) -> List[List[float]]:
63
+ out: List[List[float]] = []
64
+ for text in texts:
65
+ vec = [0.0] * self.dimension
66
+ for feat in self._features(text):
67
+ h = hashlib.blake2b(feat.encode("utf-8"), digest_size=8).digest()
68
+ idx = int.from_bytes(h[:4], "little") % self.dimension
69
+ sign = 1.0 if h[4] & 1 else -1.0
70
+ vec[idx] += sign
71
+ norm = math.sqrt(sum(v * v for v in vec))
72
+ out.append([v / norm for v in vec] if norm > 0 else vec)
73
+ return out
74
+
75
+
76
+ def openai_compat_embedder(base_url: str, model: str, **kwargs: Any) -> TextEmbedder:
77
+ from abstractmemory import OpenAICompatTextEmbedder
78
+
79
+ return OpenAICompatTextEmbedder(base_url, model, **kwargs)
80
+
81
+
82
+ class _PinnedGatewayEmbedder:
83
+ """Gateway embedder with an explicit model id (the gateway route does not report one)."""
84
+
85
+ def __init__(self, inner: Any, model_id: Optional[str]) -> None:
86
+ self._inner = inner
87
+ if model_id:
88
+ self.model_id = model_id
89
+
90
+ def embed_texts(self, texts: Sequence[str]) -> List[List[float]]:
91
+ return self._inner.embed_texts(texts)
92
+
93
+
94
+ def gateway_embedder(base_url: str, auth_token: Optional[str] = None, *, model_id: Optional[str] = None,
95
+ **kwargs: Any) -> TextEmbedder:
96
+ from abstractmemory import AbstractGatewayTextEmbedder
97
+
98
+ return _PinnedGatewayEmbedder(AbstractGatewayTextEmbedder(base_url=base_url, auth_token=auth_token, **kwargs),
99
+ model_id)
100
+
101
+
102
+ def embedder_model_id(embedder: Any) -> Optional[str]:
103
+ for attr in ("model_id", "model"):
104
+ v = getattr(embedder, attr, None)
105
+ if isinstance(v, str) and v.strip():
106
+ return v.strip()
107
+ return None
108
+
109
+
110
+ def embedder_for_pin(pin: Optional[Dict[str, Any]]) -> Optional[TextEmbedder]:
111
+ """The embedder a pin can be reopened with without configuration (hashing only)."""
112
+ if not pin:
113
+ return None
114
+ mid = str(pin.get("model_id") or "")
115
+ if mid.startswith(HASHING_MODEL_PREFIX + "-"):
116
+ try:
117
+ return HashingEmbedder(int(mid.rsplit("-", 1)[1]))
118
+ except ValueError:
119
+ return None
120
+ return None
121
+
122
+
123
+ # ----------------------------------------------------------------------------- vectors
124
+
125
+ def pack(vector: Sequence[float]) -> bytes:
126
+ return struct.pack(f"<{len(vector)}f", *[float(x) for x in vector])
127
+
128
+
129
+ def unpack(blob: bytes) -> List[float]:
130
+ n = len(blob) // 4
131
+ return list(struct.unpack(f"<{n}f", blob))
132
+
133
+
134
+ def cosine(a: Sequence[float], b: Sequence[float]) -> float:
135
+ if len(a) != len(b) or not a:
136
+ return 0.0
137
+ dot = sum(x * y for x, y in zip(a, b))
138
+ na = math.sqrt(sum(x * x for x in a))
139
+ nb = math.sqrt(sum(y * y for y in b))
140
+ return dot / (na * nb) if na > 0 and nb > 0 else 0.0
141
+
142
+
143
+ def top_k(query: Sequence[float], items: Sequence[Tuple[str, bytes]], k: int,
144
+ min_score: Optional[float] = None) -> List[Tuple[str, float]]:
145
+ """Cosine top-k of (key, packed-vector) items, best first."""
146
+ if not items or k <= 0:
147
+ return []
148
+ if _np is not None:
149
+ q = _np.asarray(query, dtype=_np.float32)
150
+ qn = float(_np.linalg.norm(q))
151
+ if qn == 0:
152
+ return []
153
+ keys = [key for key, _ in items]
154
+ mat = _np.frombuffer(b"".join(blob for _, blob in items), dtype="<f4").reshape(len(items), -1)
155
+ if mat.shape[1] != q.shape[0]:
156
+ raise ValueError("A stored vector has a different dimension from the query; rebuild with --reembed.")
157
+ norms = _np.linalg.norm(mat, axis=1)
158
+ norms[norms == 0] = 1.0
159
+ scores = (mat @ q) / (norms * qn)
160
+ order = _np.argsort(-scores)
161
+ out: List[Tuple[str, float]] = []
162
+ for idx in order[: max(k, 0) if min_score is None else len(order)]:
163
+ s = float(scores[idx])
164
+ if min_score is not None and s < min_score:
165
+ break
166
+ out.append((keys[idx], s))
167
+ if len(out) >= k:
168
+ break
169
+ return out
170
+ scored = [(key, cosine(query, unpack(blob))) for key, blob in items]
171
+ scored.sort(key=lambda t: -t[1])
172
+ if min_score is not None:
173
+ scored = [t for t in scored if t[1] >= min_score]
174
+ return scored[:k]
175
+
176
+
177
+ def vector_key(text: str, model_id: str) -> str:
178
+ return hashlib.sha256(f"{model_id}\x1f{text}".encode("utf-8")).hexdigest()
179
+
180
+
181
+ class VectorCache:
182
+ """Embeds through the notebook's embedder once per distinct (model, text).
183
+
184
+ Vectors live in the note index ``vectors`` table; only misses reach the embedder."""
185
+
186
+ def __init__(self, conn: sqlite3.Connection, lock: threading.RLock, embedder: Optional[TextEmbedder],
187
+ model_id: Optional[str], dimension: Optional[int]) -> None:
188
+ self._conn = conn
189
+ self._lock = lock
190
+ self.embedder = embedder
191
+ self.model_id = model_id or (embedder_model_id(embedder) if embedder is not None else None) or "unknown"
192
+ self.dimension = dimension
193
+ self.missing_embedder = "This notebook has no embedder, so new text cannot be embedded."
194
+
195
+ def keys_for(self, texts: Sequence[str]) -> List[str]:
196
+ return [vector_key(t, self.model_id) for t in texts]
197
+
198
+ def ensure(self, texts: Sequence[str]) -> List[str]:
199
+ """Make sure every text has a stored vector; return their keys."""
200
+ keys = self.keys_for(texts)
201
+ with self._lock:
202
+ have = set()
203
+ for i in range(0, len(keys), 500):
204
+ chunk = keys[i : i + 500]
205
+ q = ",".join("?" for _ in chunk)
206
+ have.update(r[0] for r in self._conn.execute(
207
+ f"SELECT vector_key FROM vectors WHERE vector_key IN ({q})", chunk))
208
+ missing: Dict[str, str] = {}
209
+ for key, text in zip(keys, texts):
210
+ if key not in have and key not in missing:
211
+ missing[key] = text
212
+ if missing:
213
+ if self.embedder is None:
214
+ raise ValueError(self.missing_embedder)
215
+ texts_missing = list(missing.values())
216
+ vectors = self.embedder.embed_texts(texts_missing)
217
+ if len(vectors) != len(texts_missing):
218
+ raise RuntimeError("The embedder returned a different number of vectors than texts.")
219
+ dims = {len(v) for v in vectors}
220
+ if len(dims) != 1:
221
+ raise RuntimeError("The embedder returned vectors of different dimensions.")
222
+ dim = dims.pop()
223
+ if self.dimension is not None and dim != self.dimension:
224
+ raise RuntimeError(
225
+ f"The embedder returned {dim}-dimension vectors but this notebook is pinned to "
226
+ f"{self.dimension}; rebuild with --reembed after changing the embedding model."
227
+ )
228
+ self.dimension = dim
229
+ with self._lock:
230
+ self._conn.executemany(
231
+ "INSERT OR IGNORE INTO vectors(vector_key, model_id, dimension, vector) VALUES (?,?,?,?)",
232
+ [(k, self.model_id, dim, pack(v)) for k, v in zip(missing.keys(), vectors)],
233
+ )
234
+ return keys
235
+
236
+ def get(self, key: Optional[str]) -> Optional[List[float]]:
237
+ if not key:
238
+ return None
239
+ with self._lock:
240
+ row = self._conn.execute("SELECT vector FROM vectors WHERE vector_key=?", (key,)).fetchone()
241
+ return unpack(row[0]) if row else None
242
+
243
+ def embed_queries(self, texts: Sequence[str]) -> List[Optional[List[float]]]:
244
+ """Vectors for several query texts in one embedder call (stored vectors reused; nothing stored)."""
245
+ out: List[Optional[List[float]]] = [self.get(vector_key(t, self.model_id)) for t in texts]
246
+ missing = [i for i, v in enumerate(out) if v is None]
247
+ if missing and self.embedder is not None:
248
+ vecs = self.embedder.embed_texts([texts[i] for i in missing])
249
+ for i, v in zip(missing, vecs):
250
+ v = [float(x) for x in v]
251
+ if self.dimension is not None and len(v) != self.dimension:
252
+ raise RuntimeError(
253
+ f"The embedder returned a {len(v)}-dimension vector but this notebook is pinned to "
254
+ f"{self.dimension}; rebuild with --reembed after changing the embedding model."
255
+ )
256
+ out[i] = v
257
+ return out
258
+
259
+ def embed_query(self, text: str) -> Optional[List[float]]:
260
+ """Vector for a query text: a stored vector when the text was embedded
261
+ before, else one embedder call (queries are not stored). None when the
262
+ notebook has no embedder."""
263
+ cached = self.get(vector_key(text, self.model_id))
264
+ if cached is not None:
265
+ return cached
266
+ if self.embedder is None:
267
+ return None
268
+ [vec] = self.embedder.embed_texts([text])
269
+ vec = [float(x) for x in vec]
270
+ if self.dimension is not None and len(vec) != self.dimension:
271
+ raise RuntimeError(
272
+ f"The embedder returned a {len(vec)}-dimension vector but this notebook is pinned to "
273
+ f"{self.dimension}; rebuild with --reembed after changing the embedding model."
274
+ )
275
+ return vec
@@ -0,0 +1,219 @@
1
+ """The AbstractMemory projection: ``ClaimStore`` over ``kg.sqlite3``.
2
+
3
+ Every claim becomes a ``TripleAssertion`` (``assertion_id = claim_id``); entity
4
+ names, aliases and merges become ``schema:name``, ``skos:altLabel`` and
5
+ ``schema:sameAs`` assertions. The triple store is append-only: the only
6
+ in-place change is ``close_assertion(assertion_id, valid_until)`` (an additive
7
+ AbstractMemory method), always paired with a journal ``ClosureRecord`` so that
8
+ ``TripleQuery(active_at=...)`` and ranked retrieval both agree with the note
9
+ index. Every query passes ``owner_id="abstractnote"``.
10
+
11
+ Vectors are never computed here: the note index embeds each canonical text
12
+ once (``VectorCache``) and the triple store reads them through
13
+ :class:`_LookupEmbedder`, so a rebuild does not call the embedder.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import sqlite3
19
+ import warnings
20
+ from pathlib import Path
21
+ from typing import Any, Dict, List, Optional, Sequence, Tuple
22
+
23
+ from abstractmemory import ClosureRecord, SQLiteJournal, SQLiteTripleStore, TripleAssertion, TripleQuery
24
+ from abstractmemory.canonical_text import canonical_text
25
+
26
+ from .. import ids
27
+ from ..model import CERTAINTY_CONFIDENCE
28
+ from .embeddings import VectorCache, vector_key
29
+
30
+ OWNER_ID = "abstractnote"
31
+ SCOPE = "global"
32
+ KIND_ATTR = "abstractnote_kind" # claim | name | alias | same_as
33
+
34
+
35
+ class _LookupEmbedder:
36
+ """Serves vectors the note index already computed; a miss is an error."""
37
+
38
+ def __init__(self, cache: VectorCache) -> None:
39
+ self._cache = cache
40
+ self.model_id = cache.model_id
41
+
42
+ def embed_texts(self, texts: Sequence[str]) -> List[List[float]]:
43
+ out = []
44
+ for t in texts:
45
+ v = self._cache.get(vector_key(t, self._cache.model_id))
46
+ if v is None:
47
+ raise KeyError("A triple text has no vector in the note index (embed it first).")
48
+ out.append(v)
49
+ return out
50
+
51
+
52
+ def claim_assertion(payload: Dict[str, Any], rev: Dict[str, Any], subject_kind: Optional[str],
53
+ object_kind: Optional[str], *, run_id: Optional[str] = None) -> TripleAssertion:
54
+ """The ``TripleAssertion`` of one ``claim.add`` payload (docs/data-model.md, projection table)."""
55
+ literal = payload.get("object_id") is None
56
+ attrs: Dict[str, Any] = {
57
+ KIND_ATTR: "claim", "certainty": payload["certainty"], "stance": payload.get("stance") or "assert",
58
+ }
59
+ if subject_kind:
60
+ attrs["subject_type"] = subject_kind
61
+ if object_kind and not literal:
62
+ attrs["object_type"] = object_kind
63
+ if literal:
64
+ attrs["literal"] = True
65
+ if payload.get("evidence"):
66
+ attrs["evidence_quote"] = payload["evidence"]
67
+ prov = {
68
+ "source": "abstractnote", "note_id": payload.get("note_id") or rev.get("note_id"),
69
+ "revision_id": rev["revision_id"], "run_id": run_id or rev.get("run_id"),
70
+ "evidence_quote": payload.get("evidence"), "recorded_at": rev["recorded_at"],
71
+ }
72
+ return TripleAssertion(
73
+ subject=payload["subject_id"], predicate=payload["predicate"],
74
+ object=payload["object_literal"] if literal else payload["object_id"],
75
+ scope=SCOPE, owner_id=OWNER_ID, observed_at=payload["observed_at"],
76
+ valid_from=payload.get("valid_from"), valid_until=payload.get("valid_until"),
77
+ confidence=CERTAINTY_CONFIDENCE.get(payload["certainty"], 0.5),
78
+ provenance={k: v for k, v in prov.items() if v is not None}, attributes=attrs,
79
+ assertion_id=payload["claim_id"],
80
+ )
81
+
82
+
83
+ def label_assertion(entity_id: str, predicate: str, value: str, kind: str, rev: Dict[str, Any]) -> TripleAssertion:
84
+ literal = kind in ("name", "alias")
85
+ attrs: Dict[str, Any] = {KIND_ATTR: kind}
86
+ if literal:
87
+ attrs["literal"] = True
88
+ return TripleAssertion(
89
+ subject=entity_id, predicate=predicate, object=value, scope=SCOPE, owner_id=OWNER_ID,
90
+ observed_at=rev["at"], valid_from=rev["at"] if kind == "same_as" else None,
91
+ provenance={"source": "abstractnote", "revision_id": rev["revision_id"], "recorded_at": rev["recorded_at"],
92
+ **({"note_id": rev["note_id"]} if rev.get("note_id") else {})},
93
+ attributes=attrs, assertion_id=ids.kg_assertion_id(entity_id, predicate, value),
94
+ )
95
+
96
+
97
+ def _journal_time(ts: str) -> str:
98
+ return ts[:-1] + "+00:00" if ts.endswith("Z") else ts
99
+
100
+
101
+ class ClaimStore:
102
+ """AbstractMemory ``SQLiteTripleStore`` + ``SQLiteJournal`` for one notebook."""
103
+
104
+ def __init__(self, path: Path, cache: VectorCache) -> None:
105
+ self.path = Path(path)
106
+ self.cache = cache
107
+ pin: Dict[str, Any] = {"model_id": cache.model_id, "source": "creation"}
108
+ if cache.dimension is not None:
109
+ pin["dimension"] = cache.dimension
110
+ self._embedder = _LookupEmbedder(cache)
111
+ self.store = SQLiteTripleStore(self.path, embedder=self._embedder, embedding_pin=pin)
112
+ self.journal = SQLiteJournal(self.path)
113
+
114
+ def close(self) -> None:
115
+ self.store.close()
116
+ self.journal.close()
117
+
118
+ # ------------------------------------------------------------------ projection
119
+
120
+ def plan(self, rev: Dict[str, Any], kinds: Dict[str, str]) -> Tuple[List[TripleAssertion], List[Dict[str, Any]]]:
121
+ """The assertions to add and the closures to apply for one revision."""
122
+ adds: List[TripleAssertion] = []
123
+ closes: List[Dict[str, Any]] = []
124
+ for op in rev.get("ops") or []:
125
+ name, p = op["op"], op.get("payload") or {}
126
+ if name == "entity.create":
127
+ adds.append(label_assertion(p["entity_id"], "schema:name", p["name"], "name", rev))
128
+ for alias in p.get("aliases") or []:
129
+ if str(alias).strip() and ids.fold(alias) != ids.fold(p["name"]):
130
+ adds.append(label_assertion(p["entity_id"], "skos:altLabel", str(alias).strip(), "alias", rev))
131
+ elif name == "entity.alias":
132
+ if str(p.get("alias") or "").strip():
133
+ adds.append(label_assertion(p["entity_id"], "skos:altLabel", str(p["alias"]).strip(), "alias", rev))
134
+ elif name == "entity.rename":
135
+ adds.append(label_assertion(p["entity_id"], "schema:name", p["name"], "name", rev))
136
+ if p.get("previous"):
137
+ adds.append(label_assertion(p["entity_id"], "skos:altLabel", p["previous"], "alias", rev))
138
+ elif name == "entity.merge":
139
+ adds.append(label_assertion(p["from_id"], "schema:sameAs", p["into_id"], "same_as", rev))
140
+ elif name == "entity.unmerge":
141
+ closes.append({"assertion_id": ids.kg_assertion_id(p["from_id"], "schema:sameAs", p["entity_id"]),
142
+ "valid_until": rev["at"], "kind": "retract", "reason": p.get("reason") or "unmerged",
143
+ "replacement_ids": ()})
144
+ elif name == "claim.add":
145
+ adds.append(claim_assertion(p, rev, kinds.get(p["subject_id"]),
146
+ kinds.get(p.get("object_id") or "")))
147
+ elif name == "claim.supersede":
148
+ closes.append({"assertion_id": p["claim_id"], "valid_until": p["valid_until"], "kind": "supersede",
149
+ "reason": p.get("reason") or "superseded by a later note",
150
+ "replacement_ids": (p["by_claim_id"],) if p.get("by_claim_id") else ()})
151
+ elif name == "claim.retract":
152
+ closes.append({"assertion_id": p["claim_id"], "valid_until": p.get("valid_until") or rev["at"],
153
+ "kind": "retract", "reason": p.get("reason") or "retracted", "replacement_ids": ()})
154
+ return adds, closes
155
+
156
+ def texts_for(self, adds: Sequence[TripleAssertion]) -> List[str]:
157
+ return [canonical_text(a) for a in adds]
158
+
159
+ def project(self, rev: Dict[str, Any], kinds: Dict[str, str]) -> Dict[str, int]:
160
+ """Write one revision to AbstractMemory. Idempotent: ids are deterministic,
161
+ ``add`` ignores known ids, ``close_assertion`` only closes open windows and
162
+ journal closures with a known id are no-ops."""
163
+ adds, closes = self.plan(rev, kinds)
164
+ if adds:
165
+ self.cache.ensure(self.texts_for(adds))
166
+ with warnings.catch_warnings():
167
+ # A vector miss must fail the write, never degrade it to vectorless.
168
+ warnings.filterwarnings("error", message=r"#FALLBACK: embedder failed", category=RuntimeWarning)
169
+ self.store.add(adds)
170
+ closed = 0
171
+ for c in closes:
172
+ if c["kind"] == "supersede" and not c["replacement_ids"]:
173
+ c = dict(c, kind="retract")
174
+ if self.store.close_assertion(c["assertion_id"], c["valid_until"]):
175
+ closed += 1
176
+ self.journal.append_closure(ClosureRecord(
177
+ assertion_id=c["assertion_id"], kind=c["kind"], reason=c["reason"],
178
+ replacement_ids=tuple(c["replacement_ids"]), observed_at=_journal_time(rev["recorded_at"]),
179
+ actor="abstractnote",
180
+ provenance={"revision_id": rev["revision_id"], "note_id": rev.get("note_id"),
181
+ "valid_until": c["valid_until"]},
182
+ closure_id=ids.closure_id(rev["revision_id"], c["assertion_id"], c["kind"]),
183
+ ))
184
+ return {"added": len(adds), "closed": closed}
185
+
186
+ # ------------------------------------------------------------------ reads
187
+
188
+ def query(self, **kwargs: Any) -> List[TripleAssertion]:
189
+ kwargs.setdefault("owner_id", OWNER_ID)
190
+ kwargs.setdefault("scope", SCOPE)
191
+ kwargs.setdefault("limit", 0)
192
+ return self.store.query(TripleQuery(**kwargs))
193
+
194
+ def active_claim_ids(self, at: str) -> List[str]:
195
+ """Claim assertion ids AbstractMemory considers active at ``at``."""
196
+ rows = self.query(active_at=ids.normalize_time(at, field="at"))
197
+ return sorted(a.assertion_id for a in rows if (a.attributes or {}).get(KIND_ATTR) == "claim")
198
+
199
+ def count(self) -> int:
200
+ return len(self.query())
201
+
202
+ def embedding_pin(self) -> Optional[dict]:
203
+ return self.store.embedding_pin()
204
+
205
+
206
+ def kinds_from_index(conn: sqlite3.Connection, rev: Dict[str, Any]) -> Dict[str, str]:
207
+ """Entity kinds needed to type a revision's claim assertions."""
208
+ wanted = set()
209
+ for op in rev.get("ops") or []:
210
+ if op["op"] == "claim.add":
211
+ p = op.get("payload") or {}
212
+ wanted.add(p["subject_id"])
213
+ if p.get("object_id"):
214
+ wanted.add(p["object_id"])
215
+ if not wanted:
216
+ return {}
217
+ q = ",".join("?" for _ in wanted)
218
+ return {r[0]: r[1] for r in conn.execute(f"SELECT entity_id, kind FROM entities WHERE entity_id IN ({q})",
219
+ list(wanted))}