abstractnote 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- abstractnote/__init__.py +18 -0
- abstractnote/cli.py +240 -0
- abstractnote/flows/__init__.py +2 -0
- abstractnote/flows/build.py +970 -0
- abstractnote/flows/bundle.py +62 -0
- abstractnote/flows/contract.py +88 -0
- abstractnote/flows/graph.py +540 -0
- abstractnote/flows/json/note-capture.json +2280 -0
- abstractnote/flows/json/note-drain.json +2012 -0
- abstractnote/flows/json/note-mine.json +2392 -0
- abstractnote/flows/json/note-process.json +1853 -0
- abstractnote/flows/json/note-recall.json +1876 -0
- abstractnote/flows/json/note-reflect.json +2057 -0
- abstractnote/flows/prompts_fallback.py +119 -0
- abstractnote/flows/runner.py +320 -0
- abstractnote/flows/schemas_fallback.py +175 -0
- abstractnote/gateway/__init__.py +24 -0
- abstractnote/gateway/appspec.py +10 -0
- abstractnote/gateway/router.py +571 -0
- abstractnote/gateway/tools.py +74 -0
- abstractnote/ids.py +177 -0
- abstractnote/importers/__init__.py +142 -0
- abstractnote/importers/__main__.py +3 -0
- abstractnote/importers/agora.py +203 -0
- abstractnote/importers/claude_sessions.py +415 -0
- abstractnote/importers/cli.py +113 -0
- abstractnote/importers/files.py +72 -0
- abstractnote/importers/markdown.py +165 -0
- abstractnote/ingest/__init__.py +7 -0
- abstractnote/ingest/blobs.py +47 -0
- abstractnote/ingest/capture.py +265 -0
- abstractnote/ingest/extract.py +118 -0
- abstractnote/ingest/normalise.py +95 -0
- abstractnote/model.py +229 -0
- abstractnote/process/__init__.py +5 -0
- abstractnote/process/context.py +218 -0
- abstractnote/process/merge.py +60 -0
- abstractnote/process/prompts/__init__.py +39 -0
- abstractnote/process/prompts/extract.md +30 -0
- abstractnote/process/prompts/link.md +32 -0
- abstractnote/process/prompts/mine_dossier.md +13 -0
- abstractnote/process/prompts/mine_findings.md +13 -0
- abstractnote/process/prompts/mine_plan.md +13 -0
- abstractnote/process/prompts/quick_filing.md +23 -0
- abstractnote/process/prompts/reflect.md +10 -0
- abstractnote/process/prompts/synthesize.md +17 -0
- abstractnote/process/resolve.py +96 -0
- abstractnote/process/revise.py +410 -0
- abstractnote/process/schemas.py +377 -0
- abstractnote/process/vocabulary.py +68 -0
- abstractnote/recall/__init__.py +1 -0
- abstractnote/recall/changes.py +191 -0
- abstractnote/recall/expand.py +379 -0
- abstractnote/recall/insights.py +62 -0
- abstractnote/recall/reflect.py +202 -0
- abstractnote/recall/review.py +191 -0
- abstractnote/recall/search.py +205 -0
- abstractnote/recall/shapes.py +52 -0
- abstractnote/recall/then_now.py +100 -0
- abstractnote/store/__init__.py +12 -0
- abstractnote/store/archive.py +166 -0
- abstractnote/store/embeddings.py +275 -0
- abstractnote/store/kg.py +219 -0
- abstractnote/store/notebook.py +829 -0
- abstractnote/store/notes.py +1190 -0
- abstractnote/store/queue.py +159 -0
- abstractnote/store/schema.py +348 -0
- abstractnote/store/vector_index.py +88 -0
- abstractnote/tools/__init__.py +215 -0
- abstractnote/tools/capture.py +121 -0
- abstractnote/tools/context.py +19 -0
- abstractnote/tools/process.py +22 -0
- abstractnote/tools/queue.py +23 -0
- abstractnote/tools/recall.py +64 -0
- abstractnote-0.1.0.dist-info/METADATA +145 -0
- abstractnote-0.1.0.dist-info/RECORD +80 -0
- abstractnote-0.1.0.dist-info/WHEEL +5 -0
- abstractnote-0.1.0.dist-info/entry_points.txt +2 -0
- abstractnote-0.1.0.dist-info/licenses/LICENSE +21 -0
- abstractnote-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""The archive as it stood on a date: the third time axis (docs/data-model.md "Time").
|
|
2
|
+
|
|
3
|
+
``as_received = R`` answers "what did my archive hold on R?": only the notes received by R
|
|
4
|
+
(``notes.received_at <= R``) count, so material imported later is left out even when its own dates are
|
|
5
|
+
older. Everything derived from notes follows them: a revision belongs to the archive at R when the note
|
|
6
|
+
that caused it was received by R, and a user action (no note) when it was recorded by R. Entities exist
|
|
7
|
+
when a revision in the archive created them or a note in the archive mentions them; merges, names,
|
|
8
|
+
aliases, summaries and claim closures count only when their revision belongs; a claim closed by a later
|
|
9
|
+
revision is read as it was before that revision (replayed from the revisions that do belong).
|
|
10
|
+
|
|
11
|
+
``at`` (history time) composes with it: the graph or card "as of T, from what the archive held on R".
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
18
|
+
|
|
19
|
+
from .. import ids
|
|
20
|
+
|
|
21
|
+
FOREVER = "9999-12-31T23:59:59Z"
|
|
22
|
+
|
|
23
|
+
NOTES_SQL = "SELECT note_id FROM notes WHERE received_at<=?"
|
|
24
|
+
REVISIONS_SQL = ("SELECT r.revision_id FROM revisions r LEFT JOIN notes n ON n.note_id=r.note_id"
|
|
25
|
+
" WHERE r.applied_index=1 AND COALESCE(n.received_at, r.recorded_at)<=?")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Archive:
|
|
29
|
+
"""What the archive held at ``as_received``; reads ask it which rows belong."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, store: Any, as_received: str) -> None:
|
|
32
|
+
self.store = store
|
|
33
|
+
self.at = ids.normalize_time(as_received, field="as_received")
|
|
34
|
+
self.revisions = {r[0] for r in store.q(REVISIONS_SQL, (self.at,))}
|
|
35
|
+
self._merges: Optional[Dict[str, List[Tuple[str, str, Optional[str]]]]] = None
|
|
36
|
+
self._claim_states: Dict[str, Dict[str, Any]] = {}
|
|
37
|
+
|
|
38
|
+
# ---------------------------------------------------------------- SQL filters
|
|
39
|
+
def notes_in(self, col: str) -> Tuple[str, List[Any]]:
|
|
40
|
+
"""``col IN (notes received by R)``."""
|
|
41
|
+
return f"{col} IN ({NOTES_SQL})", [self.at]
|
|
42
|
+
|
|
43
|
+
def revisions_in(self, col: str) -> Tuple[str, List[Any]]:
|
|
44
|
+
"""``col IN (revisions that belong to the archive at R)``."""
|
|
45
|
+
return f"{col} IN ({REVISIONS_SQL})", [self.at]
|
|
46
|
+
|
|
47
|
+
def has(self, revision_id: Optional[str]) -> bool:
|
|
48
|
+
return bool(revision_id) and revision_id in self.revisions
|
|
49
|
+
|
|
50
|
+
# ---------------------------------------------------------------- merges
|
|
51
|
+
def merges(self) -> Dict[str, List[Tuple[str, str, Optional[str]]]]:
|
|
52
|
+
"""``from_id -> [(at, into_id, undone_at or None)]`` for the merges in the archive, newest first; an undo
|
|
53
|
+
counts only when its own revision belongs."""
|
|
54
|
+
if self._merges is None:
|
|
55
|
+
out: Dict[str, List[Tuple[str, str, Optional[str]]]] = {}
|
|
56
|
+
for r in self.store.q("SELECT * FROM merges ORDER BY at DESC, revision_id DESC"):
|
|
57
|
+
if r["revision_id"] not in self.revisions:
|
|
58
|
+
continue
|
|
59
|
+
undone = r["undone_at"] if self.has(r["undone_revision_id"]) else None
|
|
60
|
+
out.setdefault(r["from_id"], []).append((r["at"], r["into_id"], undone))
|
|
61
|
+
self._merges = out
|
|
62
|
+
return self._merges
|
|
63
|
+
|
|
64
|
+
def canonical(self, entity_id: str, at: Optional[str] = None) -> str:
|
|
65
|
+
"""The surviving entity as of history time ``at`` (default: any time), by the merges in the archive."""
|
|
66
|
+
ref = at or FOREVER
|
|
67
|
+
merges = self.merges()
|
|
68
|
+
seen = set()
|
|
69
|
+
cur = entity_id
|
|
70
|
+
while cur not in seen:
|
|
71
|
+
seen.add(cur)
|
|
72
|
+
nxt = None
|
|
73
|
+
for m_at, into, undone in merges.get(cur, []):
|
|
74
|
+
if m_at <= ref and (undone is None or undone > ref):
|
|
75
|
+
nxt = into
|
|
76
|
+
break
|
|
77
|
+
if nxt is None:
|
|
78
|
+
return cur
|
|
79
|
+
cur = nxt
|
|
80
|
+
return cur
|
|
81
|
+
|
|
82
|
+
def members(self, entity_id: str, at: Optional[str] = None) -> List[str]:
|
|
83
|
+
"""The entity and every entry merged into it (transitively) in the archive as of ``at``."""
|
|
84
|
+
out = [entity_id]
|
|
85
|
+
for m in sorted(self.merges()):
|
|
86
|
+
if m != entity_id and self.canonical(m, at) == entity_id:
|
|
87
|
+
out.append(m)
|
|
88
|
+
return out
|
|
89
|
+
|
|
90
|
+
# ---------------------------------------------------------------- entities
|
|
91
|
+
def entity_stats(self) -> Dict[str, Dict[str, Any]]:
|
|
92
|
+
"""``entity_id -> {first_seen, last_seen, notes}`` of each entity in the archive, from its own creation
|
|
93
|
+
(when the creating revision belongs) and its own notes in the archive (``notes`` is a set of note ids)."""
|
|
94
|
+
out: Dict[str, Dict[str, Any]] = {}
|
|
95
|
+
sql, params = self.revisions_in("created_revision_id")
|
|
96
|
+
for r in self.store.q(f"SELECT entity_id, created_at FROM entities WHERE {sql}", params):
|
|
97
|
+
out[r[0]] = {"first_seen": r[1], "last_seen": r[1], "notes": set()}
|
|
98
|
+
sql, params = self.notes_in("note_id")
|
|
99
|
+
for r in self.store.q(f"SELECT entity_id, note_id, at FROM entity_notes WHERE {sql}", params):
|
|
100
|
+
st = out.setdefault(r[0], {"first_seen": r[2], "last_seen": r[2], "notes": set()})
|
|
101
|
+
st["notes"].add(r[1])
|
|
102
|
+
st["first_seen"] = min(st["first_seen"], r[2])
|
|
103
|
+
st["last_seen"] = max(st["last_seen"], r[2])
|
|
104
|
+
return out
|
|
105
|
+
|
|
106
|
+
def holds_entity(self, entity_id: str) -> bool:
|
|
107
|
+
sql, params = self.revisions_in("created_revision_id")
|
|
108
|
+
if self.store.q1(f"SELECT 1 FROM entities WHERE entity_id=? AND {sql}", [entity_id, *params]):
|
|
109
|
+
return True
|
|
110
|
+
sql, params = self.notes_in("note_id")
|
|
111
|
+
return self.store.q1(f"SELECT 1 FROM entity_notes WHERE entity_id=? AND {sql}", [entity_id, *params]) is not None
|
|
112
|
+
|
|
113
|
+
def name(self, entity_id: str, at: Optional[str] = None) -> Optional[str]:
|
|
114
|
+
sql, params = self.revisions_in("revision_id")
|
|
115
|
+
r = self.store.q1(f"SELECT name FROM entity_names WHERE entity_id=? AND at<=? AND {sql} ORDER BY at DESC,"
|
|
116
|
+
" revision_id DESC LIMIT 1", [entity_id, at or FOREVER, *params])
|
|
117
|
+
return r[0] if r else None
|
|
118
|
+
|
|
119
|
+
def summary(self, entity_id: str, at: Optional[str] = None) -> Optional[str]:
|
|
120
|
+
sql, params = self.revisions_in("revision_id")
|
|
121
|
+
r = self.store.q1(f"SELECT summary FROM entity_summaries WHERE entity_id=? AND at<=? AND {sql} ORDER BY at"
|
|
122
|
+
" DESC, revision_id DESC LIMIT 1", [entity_id, at or FOREVER, *params])
|
|
123
|
+
return r[0] if r else None
|
|
124
|
+
|
|
125
|
+
# ---------------------------------------------------------------- claims
|
|
126
|
+
def claim_state(self, row: Any) -> Dict[str, Any]:
|
|
127
|
+
"""``{status, valid_until, superseded_by}`` of a claim as the archive at R held it."""
|
|
128
|
+
cid = row["claim_id"]
|
|
129
|
+
if not row["closed_revision_id"] or row["closed_revision_id"] in self.revisions:
|
|
130
|
+
return {"status": row["status"], "valid_until": row["valid_until"], "superseded_by": row["superseded_by"]}
|
|
131
|
+
if cid in self._claim_states:
|
|
132
|
+
return self._claim_states[cid]
|
|
133
|
+
valid_until = None
|
|
134
|
+
added = self.store.revision_row(row["revision_id"])
|
|
135
|
+
for op in json.loads(added["ops_json"]) if added else []:
|
|
136
|
+
p = op.get("payload") or {}
|
|
137
|
+
if op.get("op") == "claim.add" and p.get("claim_id") == cid:
|
|
138
|
+
valid_until = p.get("valid_until")
|
|
139
|
+
state = {"status": "active", "valid_until": valid_until, "superseded_by": None}
|
|
140
|
+
for r in self.store.q("SELECT revision_id, at, ops_json FROM revisions WHERE applied_index=1 AND ops_json LIKE ?"
|
|
141
|
+
" ORDER BY seq", (f'%"{cid}"%',)):
|
|
142
|
+
if r["revision_id"] not in self.revisions:
|
|
143
|
+
continue
|
|
144
|
+
for op in json.loads(r["ops_json"]):
|
|
145
|
+
p = op.get("payload") or {}
|
|
146
|
+
if p.get("claim_id") != cid:
|
|
147
|
+
continue
|
|
148
|
+
if op.get("op") == "claim.supersede" and state["status"] == "active":
|
|
149
|
+
state = {"status": "superseded", "valid_until": state["valid_until"] or p.get("valid_until"),
|
|
150
|
+
"superseded_by": p.get("by_claim_id")}
|
|
151
|
+
elif op.get("op") == "claim.retract" and state["status"] != "retracted":
|
|
152
|
+
state = {"status": "retracted",
|
|
153
|
+
"valid_until": state["valid_until"] or p.get("valid_until") or r["at"],
|
|
154
|
+
"superseded_by": state["superseded_by"]}
|
|
155
|
+
self._claim_states[cid] = state
|
|
156
|
+
return state
|
|
157
|
+
|
|
158
|
+
def claim_dict(self, row: Any) -> Dict[str, Any]:
|
|
159
|
+
return {**self.store.claim_dict(row), **self.claim_state(row)}
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def archive_for(store: Any, as_received: Optional[str]) -> Optional[Archive]:
|
|
163
|
+
"""An :class:`Archive` for a non-empty ``as_received``, else None (the archive as it is now)."""
|
|
164
|
+
if as_received in (None, ""):
|
|
165
|
+
return None
|
|
166
|
+
return Archive(store, as_received)
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Embedders and the notebook's embedding cache.
|
|
2
|
+
|
|
3
|
+
One notebook = one embedding space. The pin ``{model_id, dimension}`` is kept
|
|
4
|
+
in the note index ``meta`` table and given to the AbstractMemory store at
|
|
5
|
+
creation. Every vector is computed once through :class:`VectorCache` (keyed by
|
|
6
|
+
the sha256 of the embedded text and the model id), stored in the note index,
|
|
7
|
+
and handed to the triple store, so a rebuild never re-embeds unless asked.
|
|
8
|
+
|
|
9
|
+
Adapters:
|
|
10
|
+
|
|
11
|
+
- :class:`HashingEmbedder` — deterministic feature hashing, offline; the
|
|
12
|
+
default for headless notebooks and the test suite.
|
|
13
|
+
- :func:`openai_compat_embedder` — any OpenAI-compatible ``/v1/embeddings``.
|
|
14
|
+
- :func:`gateway_embedder` — the gateway embedding route via
|
|
15
|
+
``abstractmemory.AbstractGatewayTextEmbedder``.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import hashlib
|
|
21
|
+
import math
|
|
22
|
+
import re
|
|
23
|
+
import sqlite3
|
|
24
|
+
import struct
|
|
25
|
+
import threading
|
|
26
|
+
from typing import Any, Dict, Iterable, List, Optional, Protocol, Sequence, Tuple
|
|
27
|
+
|
|
28
|
+
try: # numpy is optional: faster cosine scans when present
|
|
29
|
+
import numpy as _np
|
|
30
|
+
except Exception: # pragma: no cover
|
|
31
|
+
_np = None
|
|
32
|
+
|
|
33
|
+
HASHING_MODEL_PREFIX = "abstractnote-hashing-v1"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class TextEmbedder(Protocol):
|
|
37
|
+
def embed_texts(self, texts: Sequence[str]) -> List[List[float]]: ...
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_TOKEN_RE = re.compile(r"\w+", re.UNICODE)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class HashingEmbedder:
|
|
44
|
+
"""Deterministic, offline embedder: signed feature hashing of words and
|
|
45
|
+
character trigrams, L2-normalised. Good enough for exact and near-exact
|
|
46
|
+
similarity (tests, headless use); not a semantic model."""
|
|
47
|
+
|
|
48
|
+
def __init__(self, dimension: int = 256) -> None:
|
|
49
|
+
if dimension < 8:
|
|
50
|
+
raise ValueError("The hashing embedder needs at least 8 dimensions.")
|
|
51
|
+
self.dimension = int(dimension)
|
|
52
|
+
self.model_id = f"{HASHING_MODEL_PREFIX}-{self.dimension}"
|
|
53
|
+
|
|
54
|
+
def _features(self, text: str) -> Iterable[str]:
|
|
55
|
+
words = _TOKEN_RE.findall(str(text or "").casefold())
|
|
56
|
+
for w in words:
|
|
57
|
+
yield "w:" + w
|
|
58
|
+
padded = f"#{w}#"
|
|
59
|
+
for i in range(len(padded) - 2):
|
|
60
|
+
yield "c:" + padded[i : i + 3]
|
|
61
|
+
|
|
62
|
+
def embed_texts(self, texts: Sequence[str]) -> List[List[float]]:
|
|
63
|
+
out: List[List[float]] = []
|
|
64
|
+
for text in texts:
|
|
65
|
+
vec = [0.0] * self.dimension
|
|
66
|
+
for feat in self._features(text):
|
|
67
|
+
h = hashlib.blake2b(feat.encode("utf-8"), digest_size=8).digest()
|
|
68
|
+
idx = int.from_bytes(h[:4], "little") % self.dimension
|
|
69
|
+
sign = 1.0 if h[4] & 1 else -1.0
|
|
70
|
+
vec[idx] += sign
|
|
71
|
+
norm = math.sqrt(sum(v * v for v in vec))
|
|
72
|
+
out.append([v / norm for v in vec] if norm > 0 else vec)
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def openai_compat_embedder(base_url: str, model: str, **kwargs: Any) -> TextEmbedder:
|
|
77
|
+
from abstractmemory import OpenAICompatTextEmbedder
|
|
78
|
+
|
|
79
|
+
return OpenAICompatTextEmbedder(base_url, model, **kwargs)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _PinnedGatewayEmbedder:
|
|
83
|
+
"""Gateway embedder with an explicit model id (the gateway route does not report one)."""
|
|
84
|
+
|
|
85
|
+
def __init__(self, inner: Any, model_id: Optional[str]) -> None:
|
|
86
|
+
self._inner = inner
|
|
87
|
+
if model_id:
|
|
88
|
+
self.model_id = model_id
|
|
89
|
+
|
|
90
|
+
def embed_texts(self, texts: Sequence[str]) -> List[List[float]]:
|
|
91
|
+
return self._inner.embed_texts(texts)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def gateway_embedder(base_url: str, auth_token: Optional[str] = None, *, model_id: Optional[str] = None,
|
|
95
|
+
**kwargs: Any) -> TextEmbedder:
|
|
96
|
+
from abstractmemory import AbstractGatewayTextEmbedder
|
|
97
|
+
|
|
98
|
+
return _PinnedGatewayEmbedder(AbstractGatewayTextEmbedder(base_url=base_url, auth_token=auth_token, **kwargs),
|
|
99
|
+
model_id)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def embedder_model_id(embedder: Any) -> Optional[str]:
|
|
103
|
+
for attr in ("model_id", "model"):
|
|
104
|
+
v = getattr(embedder, attr, None)
|
|
105
|
+
if isinstance(v, str) and v.strip():
|
|
106
|
+
return v.strip()
|
|
107
|
+
return None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def embedder_for_pin(pin: Optional[Dict[str, Any]]) -> Optional[TextEmbedder]:
|
|
111
|
+
"""The embedder a pin can be reopened with without configuration (hashing only)."""
|
|
112
|
+
if not pin:
|
|
113
|
+
return None
|
|
114
|
+
mid = str(pin.get("model_id") or "")
|
|
115
|
+
if mid.startswith(HASHING_MODEL_PREFIX + "-"):
|
|
116
|
+
try:
|
|
117
|
+
return HashingEmbedder(int(mid.rsplit("-", 1)[1]))
|
|
118
|
+
except ValueError:
|
|
119
|
+
return None
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
# ----------------------------------------------------------------------------- vectors
|
|
124
|
+
|
|
125
|
+
def pack(vector: Sequence[float]) -> bytes:
|
|
126
|
+
return struct.pack(f"<{len(vector)}f", *[float(x) for x in vector])
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def unpack(blob: bytes) -> List[float]:
|
|
130
|
+
n = len(blob) // 4
|
|
131
|
+
return list(struct.unpack(f"<{n}f", blob))
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def cosine(a: Sequence[float], b: Sequence[float]) -> float:
|
|
135
|
+
if len(a) != len(b) or not a:
|
|
136
|
+
return 0.0
|
|
137
|
+
dot = sum(x * y for x, y in zip(a, b))
|
|
138
|
+
na = math.sqrt(sum(x * x for x in a))
|
|
139
|
+
nb = math.sqrt(sum(y * y for y in b))
|
|
140
|
+
return dot / (na * nb) if na > 0 and nb > 0 else 0.0
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def top_k(query: Sequence[float], items: Sequence[Tuple[str, bytes]], k: int,
|
|
144
|
+
min_score: Optional[float] = None) -> List[Tuple[str, float]]:
|
|
145
|
+
"""Cosine top-k of (key, packed-vector) items, best first."""
|
|
146
|
+
if not items or k <= 0:
|
|
147
|
+
return []
|
|
148
|
+
if _np is not None:
|
|
149
|
+
q = _np.asarray(query, dtype=_np.float32)
|
|
150
|
+
qn = float(_np.linalg.norm(q))
|
|
151
|
+
if qn == 0:
|
|
152
|
+
return []
|
|
153
|
+
keys = [key for key, _ in items]
|
|
154
|
+
mat = _np.frombuffer(b"".join(blob for _, blob in items), dtype="<f4").reshape(len(items), -1)
|
|
155
|
+
if mat.shape[1] != q.shape[0]:
|
|
156
|
+
raise ValueError("A stored vector has a different dimension from the query; rebuild with --reembed.")
|
|
157
|
+
norms = _np.linalg.norm(mat, axis=1)
|
|
158
|
+
norms[norms == 0] = 1.0
|
|
159
|
+
scores = (mat @ q) / (norms * qn)
|
|
160
|
+
order = _np.argsort(-scores)
|
|
161
|
+
out: List[Tuple[str, float]] = []
|
|
162
|
+
for idx in order[: max(k, 0) if min_score is None else len(order)]:
|
|
163
|
+
s = float(scores[idx])
|
|
164
|
+
if min_score is not None and s < min_score:
|
|
165
|
+
break
|
|
166
|
+
out.append((keys[idx], s))
|
|
167
|
+
if len(out) >= k:
|
|
168
|
+
break
|
|
169
|
+
return out
|
|
170
|
+
scored = [(key, cosine(query, unpack(blob))) for key, blob in items]
|
|
171
|
+
scored.sort(key=lambda t: -t[1])
|
|
172
|
+
if min_score is not None:
|
|
173
|
+
scored = [t for t in scored if t[1] >= min_score]
|
|
174
|
+
return scored[:k]
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def vector_key(text: str, model_id: str) -> str:
|
|
178
|
+
return hashlib.sha256(f"{model_id}\x1f{text}".encode("utf-8")).hexdigest()
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class VectorCache:
|
|
182
|
+
"""Embeds through the notebook's embedder once per distinct (model, text).
|
|
183
|
+
|
|
184
|
+
Vectors live in the note index ``vectors`` table; only misses reach the embedder."""
|
|
185
|
+
|
|
186
|
+
def __init__(self, conn: sqlite3.Connection, lock: threading.RLock, embedder: Optional[TextEmbedder],
|
|
187
|
+
model_id: Optional[str], dimension: Optional[int]) -> None:
|
|
188
|
+
self._conn = conn
|
|
189
|
+
self._lock = lock
|
|
190
|
+
self.embedder = embedder
|
|
191
|
+
self.model_id = model_id or (embedder_model_id(embedder) if embedder is not None else None) or "unknown"
|
|
192
|
+
self.dimension = dimension
|
|
193
|
+
self.missing_embedder = "This notebook has no embedder, so new text cannot be embedded."
|
|
194
|
+
|
|
195
|
+
def keys_for(self, texts: Sequence[str]) -> List[str]:
|
|
196
|
+
return [vector_key(t, self.model_id) for t in texts]
|
|
197
|
+
|
|
198
|
+
def ensure(self, texts: Sequence[str]) -> List[str]:
|
|
199
|
+
"""Make sure every text has a stored vector; return their keys."""
|
|
200
|
+
keys = self.keys_for(texts)
|
|
201
|
+
with self._lock:
|
|
202
|
+
have = set()
|
|
203
|
+
for i in range(0, len(keys), 500):
|
|
204
|
+
chunk = keys[i : i + 500]
|
|
205
|
+
q = ",".join("?" for _ in chunk)
|
|
206
|
+
have.update(r[0] for r in self._conn.execute(
|
|
207
|
+
f"SELECT vector_key FROM vectors WHERE vector_key IN ({q})", chunk))
|
|
208
|
+
missing: Dict[str, str] = {}
|
|
209
|
+
for key, text in zip(keys, texts):
|
|
210
|
+
if key not in have and key not in missing:
|
|
211
|
+
missing[key] = text
|
|
212
|
+
if missing:
|
|
213
|
+
if self.embedder is None:
|
|
214
|
+
raise ValueError(self.missing_embedder)
|
|
215
|
+
texts_missing = list(missing.values())
|
|
216
|
+
vectors = self.embedder.embed_texts(texts_missing)
|
|
217
|
+
if len(vectors) != len(texts_missing):
|
|
218
|
+
raise RuntimeError("The embedder returned a different number of vectors than texts.")
|
|
219
|
+
dims = {len(v) for v in vectors}
|
|
220
|
+
if len(dims) != 1:
|
|
221
|
+
raise RuntimeError("The embedder returned vectors of different dimensions.")
|
|
222
|
+
dim = dims.pop()
|
|
223
|
+
if self.dimension is not None and dim != self.dimension:
|
|
224
|
+
raise RuntimeError(
|
|
225
|
+
f"The embedder returned {dim}-dimension vectors but this notebook is pinned to "
|
|
226
|
+
f"{self.dimension}; rebuild with --reembed after changing the embedding model."
|
|
227
|
+
)
|
|
228
|
+
self.dimension = dim
|
|
229
|
+
with self._lock:
|
|
230
|
+
self._conn.executemany(
|
|
231
|
+
"INSERT OR IGNORE INTO vectors(vector_key, model_id, dimension, vector) VALUES (?,?,?,?)",
|
|
232
|
+
[(k, self.model_id, dim, pack(v)) for k, v in zip(missing.keys(), vectors)],
|
|
233
|
+
)
|
|
234
|
+
return keys
|
|
235
|
+
|
|
236
|
+
def get(self, key: Optional[str]) -> Optional[List[float]]:
|
|
237
|
+
if not key:
|
|
238
|
+
return None
|
|
239
|
+
with self._lock:
|
|
240
|
+
row = self._conn.execute("SELECT vector FROM vectors WHERE vector_key=?", (key,)).fetchone()
|
|
241
|
+
return unpack(row[0]) if row else None
|
|
242
|
+
|
|
243
|
+
def embed_queries(self, texts: Sequence[str]) -> List[Optional[List[float]]]:
|
|
244
|
+
"""Vectors for several query texts in one embedder call (stored vectors reused; nothing stored)."""
|
|
245
|
+
out: List[Optional[List[float]]] = [self.get(vector_key(t, self.model_id)) for t in texts]
|
|
246
|
+
missing = [i for i, v in enumerate(out) if v is None]
|
|
247
|
+
if missing and self.embedder is not None:
|
|
248
|
+
vecs = self.embedder.embed_texts([texts[i] for i in missing])
|
|
249
|
+
for i, v in zip(missing, vecs):
|
|
250
|
+
v = [float(x) for x in v]
|
|
251
|
+
if self.dimension is not None and len(v) != self.dimension:
|
|
252
|
+
raise RuntimeError(
|
|
253
|
+
f"The embedder returned a {len(v)}-dimension vector but this notebook is pinned to "
|
|
254
|
+
f"{self.dimension}; rebuild with --reembed after changing the embedding model."
|
|
255
|
+
)
|
|
256
|
+
out[i] = v
|
|
257
|
+
return out
|
|
258
|
+
|
|
259
|
+
def embed_query(self, text: str) -> Optional[List[float]]:
|
|
260
|
+
"""Vector for a query text: a stored vector when the text was embedded
|
|
261
|
+
before, else one embedder call (queries are not stored). None when the
|
|
262
|
+
notebook has no embedder."""
|
|
263
|
+
cached = self.get(vector_key(text, self.model_id))
|
|
264
|
+
if cached is not None:
|
|
265
|
+
return cached
|
|
266
|
+
if self.embedder is None:
|
|
267
|
+
return None
|
|
268
|
+
[vec] = self.embedder.embed_texts([text])
|
|
269
|
+
vec = [float(x) for x in vec]
|
|
270
|
+
if self.dimension is not None and len(vec) != self.dimension:
|
|
271
|
+
raise RuntimeError(
|
|
272
|
+
f"The embedder returned a {len(vec)}-dimension vector but this notebook is pinned to "
|
|
273
|
+
f"{self.dimension}; rebuild with --reembed after changing the embedding model."
|
|
274
|
+
)
|
|
275
|
+
return vec
|
abstractnote/store/kg.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""The AbstractMemory projection: ``ClaimStore`` over ``kg.sqlite3``.
|
|
2
|
+
|
|
3
|
+
Every claim becomes a ``TripleAssertion`` (``assertion_id = claim_id``); entity
|
|
4
|
+
names, aliases and merges become ``schema:name``, ``skos:altLabel`` and
|
|
5
|
+
``schema:sameAs`` assertions. The triple store is append-only: the only
|
|
6
|
+
in-place change is ``close_assertion(assertion_id, valid_until)`` (an additive
|
|
7
|
+
AbstractMemory method), always paired with a journal ``ClosureRecord`` so that
|
|
8
|
+
``TripleQuery(active_at=...)`` and ranked retrieval both agree with the note
|
|
9
|
+
index. Every query passes ``owner_id="abstractnote"``.
|
|
10
|
+
|
|
11
|
+
Vectors are never computed here: the note index embeds each canonical text
|
|
12
|
+
once (``VectorCache``) and the triple store reads them through
|
|
13
|
+
:class:`_LookupEmbedder`, so a rebuild does not call the embedder.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import sqlite3
|
|
19
|
+
import warnings
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
22
|
+
|
|
23
|
+
from abstractmemory import ClosureRecord, SQLiteJournal, SQLiteTripleStore, TripleAssertion, TripleQuery
|
|
24
|
+
from abstractmemory.canonical_text import canonical_text
|
|
25
|
+
|
|
26
|
+
from .. import ids
|
|
27
|
+
from ..model import CERTAINTY_CONFIDENCE
|
|
28
|
+
from .embeddings import VectorCache, vector_key
|
|
29
|
+
|
|
30
|
+
OWNER_ID = "abstractnote"
|
|
31
|
+
SCOPE = "global"
|
|
32
|
+
KIND_ATTR = "abstractnote_kind" # claim | name | alias | same_as
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class _LookupEmbedder:
|
|
36
|
+
"""Serves vectors the note index already computed; a miss is an error."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, cache: VectorCache) -> None:
|
|
39
|
+
self._cache = cache
|
|
40
|
+
self.model_id = cache.model_id
|
|
41
|
+
|
|
42
|
+
def embed_texts(self, texts: Sequence[str]) -> List[List[float]]:
|
|
43
|
+
out = []
|
|
44
|
+
for t in texts:
|
|
45
|
+
v = self._cache.get(vector_key(t, self._cache.model_id))
|
|
46
|
+
if v is None:
|
|
47
|
+
raise KeyError("A triple text has no vector in the note index (embed it first).")
|
|
48
|
+
out.append(v)
|
|
49
|
+
return out
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def claim_assertion(payload: Dict[str, Any], rev: Dict[str, Any], subject_kind: Optional[str],
|
|
53
|
+
object_kind: Optional[str], *, run_id: Optional[str] = None) -> TripleAssertion:
|
|
54
|
+
"""The ``TripleAssertion`` of one ``claim.add`` payload (docs/data-model.md, projection table)."""
|
|
55
|
+
literal = payload.get("object_id") is None
|
|
56
|
+
attrs: Dict[str, Any] = {
|
|
57
|
+
KIND_ATTR: "claim", "certainty": payload["certainty"], "stance": payload.get("stance") or "assert",
|
|
58
|
+
}
|
|
59
|
+
if subject_kind:
|
|
60
|
+
attrs["subject_type"] = subject_kind
|
|
61
|
+
if object_kind and not literal:
|
|
62
|
+
attrs["object_type"] = object_kind
|
|
63
|
+
if literal:
|
|
64
|
+
attrs["literal"] = True
|
|
65
|
+
if payload.get("evidence"):
|
|
66
|
+
attrs["evidence_quote"] = payload["evidence"]
|
|
67
|
+
prov = {
|
|
68
|
+
"source": "abstractnote", "note_id": payload.get("note_id") or rev.get("note_id"),
|
|
69
|
+
"revision_id": rev["revision_id"], "run_id": run_id or rev.get("run_id"),
|
|
70
|
+
"evidence_quote": payload.get("evidence"), "recorded_at": rev["recorded_at"],
|
|
71
|
+
}
|
|
72
|
+
return TripleAssertion(
|
|
73
|
+
subject=payload["subject_id"], predicate=payload["predicate"],
|
|
74
|
+
object=payload["object_literal"] if literal else payload["object_id"],
|
|
75
|
+
scope=SCOPE, owner_id=OWNER_ID, observed_at=payload["observed_at"],
|
|
76
|
+
valid_from=payload.get("valid_from"), valid_until=payload.get("valid_until"),
|
|
77
|
+
confidence=CERTAINTY_CONFIDENCE.get(payload["certainty"], 0.5),
|
|
78
|
+
provenance={k: v for k, v in prov.items() if v is not None}, attributes=attrs,
|
|
79
|
+
assertion_id=payload["claim_id"],
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def label_assertion(entity_id: str, predicate: str, value: str, kind: str, rev: Dict[str, Any]) -> TripleAssertion:
|
|
84
|
+
literal = kind in ("name", "alias")
|
|
85
|
+
attrs: Dict[str, Any] = {KIND_ATTR: kind}
|
|
86
|
+
if literal:
|
|
87
|
+
attrs["literal"] = True
|
|
88
|
+
return TripleAssertion(
|
|
89
|
+
subject=entity_id, predicate=predicate, object=value, scope=SCOPE, owner_id=OWNER_ID,
|
|
90
|
+
observed_at=rev["at"], valid_from=rev["at"] if kind == "same_as" else None,
|
|
91
|
+
provenance={"source": "abstractnote", "revision_id": rev["revision_id"], "recorded_at": rev["recorded_at"],
|
|
92
|
+
**({"note_id": rev["note_id"]} if rev.get("note_id") else {})},
|
|
93
|
+
attributes=attrs, assertion_id=ids.kg_assertion_id(entity_id, predicate, value),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _journal_time(ts: str) -> str:
|
|
98
|
+
return ts[:-1] + "+00:00" if ts.endswith("Z") else ts
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class ClaimStore:
|
|
102
|
+
"""AbstractMemory ``SQLiteTripleStore`` + ``SQLiteJournal`` for one notebook."""
|
|
103
|
+
|
|
104
|
+
def __init__(self, path: Path, cache: VectorCache) -> None:
|
|
105
|
+
self.path = Path(path)
|
|
106
|
+
self.cache = cache
|
|
107
|
+
pin: Dict[str, Any] = {"model_id": cache.model_id, "source": "creation"}
|
|
108
|
+
if cache.dimension is not None:
|
|
109
|
+
pin["dimension"] = cache.dimension
|
|
110
|
+
self._embedder = _LookupEmbedder(cache)
|
|
111
|
+
self.store = SQLiteTripleStore(self.path, embedder=self._embedder, embedding_pin=pin)
|
|
112
|
+
self.journal = SQLiteJournal(self.path)
|
|
113
|
+
|
|
114
|
+
def close(self) -> None:
|
|
115
|
+
self.store.close()
|
|
116
|
+
self.journal.close()
|
|
117
|
+
|
|
118
|
+
# ------------------------------------------------------------------ projection
|
|
119
|
+
|
|
120
|
+
def plan(self, rev: Dict[str, Any], kinds: Dict[str, str]) -> Tuple[List[TripleAssertion], List[Dict[str, Any]]]:
|
|
121
|
+
"""The assertions to add and the closures to apply for one revision."""
|
|
122
|
+
adds: List[TripleAssertion] = []
|
|
123
|
+
closes: List[Dict[str, Any]] = []
|
|
124
|
+
for op in rev.get("ops") or []:
|
|
125
|
+
name, p = op["op"], op.get("payload") or {}
|
|
126
|
+
if name == "entity.create":
|
|
127
|
+
adds.append(label_assertion(p["entity_id"], "schema:name", p["name"], "name", rev))
|
|
128
|
+
for alias in p.get("aliases") or []:
|
|
129
|
+
if str(alias).strip() and ids.fold(alias) != ids.fold(p["name"]):
|
|
130
|
+
adds.append(label_assertion(p["entity_id"], "skos:altLabel", str(alias).strip(), "alias", rev))
|
|
131
|
+
elif name == "entity.alias":
|
|
132
|
+
if str(p.get("alias") or "").strip():
|
|
133
|
+
adds.append(label_assertion(p["entity_id"], "skos:altLabel", str(p["alias"]).strip(), "alias", rev))
|
|
134
|
+
elif name == "entity.rename":
|
|
135
|
+
adds.append(label_assertion(p["entity_id"], "schema:name", p["name"], "name", rev))
|
|
136
|
+
if p.get("previous"):
|
|
137
|
+
adds.append(label_assertion(p["entity_id"], "skos:altLabel", p["previous"], "alias", rev))
|
|
138
|
+
elif name == "entity.merge":
|
|
139
|
+
adds.append(label_assertion(p["from_id"], "schema:sameAs", p["into_id"], "same_as", rev))
|
|
140
|
+
elif name == "entity.unmerge":
|
|
141
|
+
closes.append({"assertion_id": ids.kg_assertion_id(p["from_id"], "schema:sameAs", p["entity_id"]),
|
|
142
|
+
"valid_until": rev["at"], "kind": "retract", "reason": p.get("reason") or "unmerged",
|
|
143
|
+
"replacement_ids": ()})
|
|
144
|
+
elif name == "claim.add":
|
|
145
|
+
adds.append(claim_assertion(p, rev, kinds.get(p["subject_id"]),
|
|
146
|
+
kinds.get(p.get("object_id") or "")))
|
|
147
|
+
elif name == "claim.supersede":
|
|
148
|
+
closes.append({"assertion_id": p["claim_id"], "valid_until": p["valid_until"], "kind": "supersede",
|
|
149
|
+
"reason": p.get("reason") or "superseded by a later note",
|
|
150
|
+
"replacement_ids": (p["by_claim_id"],) if p.get("by_claim_id") else ()})
|
|
151
|
+
elif name == "claim.retract":
|
|
152
|
+
closes.append({"assertion_id": p["claim_id"], "valid_until": p.get("valid_until") or rev["at"],
|
|
153
|
+
"kind": "retract", "reason": p.get("reason") or "retracted", "replacement_ids": ()})
|
|
154
|
+
return adds, closes
|
|
155
|
+
|
|
156
|
+
def texts_for(self, adds: Sequence[TripleAssertion]) -> List[str]:
|
|
157
|
+
return [canonical_text(a) for a in adds]
|
|
158
|
+
|
|
159
|
+
def project(self, rev: Dict[str, Any], kinds: Dict[str, str]) -> Dict[str, int]:
|
|
160
|
+
"""Write one revision to AbstractMemory. Idempotent: ids are deterministic,
|
|
161
|
+
``add`` ignores known ids, ``close_assertion`` only closes open windows and
|
|
162
|
+
journal closures with a known id are no-ops."""
|
|
163
|
+
adds, closes = self.plan(rev, kinds)
|
|
164
|
+
if adds:
|
|
165
|
+
self.cache.ensure(self.texts_for(adds))
|
|
166
|
+
with warnings.catch_warnings():
|
|
167
|
+
# A vector miss must fail the write, never degrade it to vectorless.
|
|
168
|
+
warnings.filterwarnings("error", message=r"#FALLBACK: embedder failed", category=RuntimeWarning)
|
|
169
|
+
self.store.add(adds)
|
|
170
|
+
closed = 0
|
|
171
|
+
for c in closes:
|
|
172
|
+
if c["kind"] == "supersede" and not c["replacement_ids"]:
|
|
173
|
+
c = dict(c, kind="retract")
|
|
174
|
+
if self.store.close_assertion(c["assertion_id"], c["valid_until"]):
|
|
175
|
+
closed += 1
|
|
176
|
+
self.journal.append_closure(ClosureRecord(
|
|
177
|
+
assertion_id=c["assertion_id"], kind=c["kind"], reason=c["reason"],
|
|
178
|
+
replacement_ids=tuple(c["replacement_ids"]), observed_at=_journal_time(rev["recorded_at"]),
|
|
179
|
+
actor="abstractnote",
|
|
180
|
+
provenance={"revision_id": rev["revision_id"], "note_id": rev.get("note_id"),
|
|
181
|
+
"valid_until": c["valid_until"]},
|
|
182
|
+
closure_id=ids.closure_id(rev["revision_id"], c["assertion_id"], c["kind"]),
|
|
183
|
+
))
|
|
184
|
+
return {"added": len(adds), "closed": closed}
|
|
185
|
+
|
|
186
|
+
# ------------------------------------------------------------------ reads
|
|
187
|
+
|
|
188
|
+
def query(self, **kwargs: Any) -> List[TripleAssertion]:
|
|
189
|
+
kwargs.setdefault("owner_id", OWNER_ID)
|
|
190
|
+
kwargs.setdefault("scope", SCOPE)
|
|
191
|
+
kwargs.setdefault("limit", 0)
|
|
192
|
+
return self.store.query(TripleQuery(**kwargs))
|
|
193
|
+
|
|
194
|
+
def active_claim_ids(self, at: str) -> List[str]:
|
|
195
|
+
"""Claim assertion ids AbstractMemory considers active at ``at``."""
|
|
196
|
+
rows = self.query(active_at=ids.normalize_time(at, field="at"))
|
|
197
|
+
return sorted(a.assertion_id for a in rows if (a.attributes or {}).get(KIND_ATTR) == "claim")
|
|
198
|
+
|
|
199
|
+
def count(self) -> int:
|
|
200
|
+
return len(self.query())
|
|
201
|
+
|
|
202
|
+
def embedding_pin(self) -> Optional[dict]:
|
|
203
|
+
return self.store.embedding_pin()
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def kinds_from_index(conn: sqlite3.Connection, rev: Dict[str, Any]) -> Dict[str, str]:
|
|
207
|
+
"""Entity kinds needed to type a revision's claim assertions."""
|
|
208
|
+
wanted = set()
|
|
209
|
+
for op in rev.get("ops") or []:
|
|
210
|
+
if op["op"] == "claim.add":
|
|
211
|
+
p = op.get("payload") or {}
|
|
212
|
+
wanted.add(p["subject_id"])
|
|
213
|
+
if p.get("object_id"):
|
|
214
|
+
wanted.add(p["object_id"])
|
|
215
|
+
if not wanted:
|
|
216
|
+
return {}
|
|
217
|
+
q = ",".join("?" for _ in wanted)
|
|
218
|
+
return {r[0]: r[1] for r in conn.execute(f"SELECT entity_id, kind FROM entities WHERE entity_id IN ({q})",
|
|
219
|
+
list(wanted))}
|