second-brain-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- second_brain_core/__init__.py +32 -0
- second_brain_core/annotations.py +15 -0
- second_brain_core/audit.py +275 -0
- second_brain_core/auth.py +235 -0
- second_brain_core/env.py +171 -0
- second_brain_core/errors.py +37 -0
- second_brain_core/failure.py +87 -0
- second_brain_core/health.py +48 -0
- second_brain_core/py.typed +0 -0
- second_brain_core/recent.py +69 -0
- second_brain_core/remote.py +252 -0
- second_brain_core/semantic/__init__.py +7 -0
- second_brain_core/semantic/chunking.py +48 -0
- second_brain_core/semantic/embedders.py +68 -0
- second_brain_core/semantic/engine.py +218 -0
- second_brain_core/semantic/stores.py +539 -0
- second_brain_core/semantic/sync.py +17 -0
- second_brain_core/semantic/testing.py +183 -0
- second_brain_core/semantic/types.py +130 -0
- second_brain_core/server.py +44 -0
- second_brain_core/testing.py +57 -0
- second_brain_core-0.1.0.dist-info/METADATA +106 -0
- second_brain_core-0.1.0.dist-info/RECORD +25 -0
- second_brain_core-0.1.0.dist-info/WHEEL +4 -0
- second_brain_core-0.1.0.dist-info/licenses/LICENSE +661 -0
|
@@ -0,0 +1,539 @@
|
|
|
1
|
+
"""The two index stores: sqlite-vec (the default) and Chroma.
|
|
2
|
+
|
|
3
|
+
Every heavy import lives inside a constructor, so a missing extra surfaces as a
|
|
4
|
+
:class:`~second_brain_core.errors.SemanticUnavailableError` that says what to install.
|
|
5
|
+
|
|
6
|
+
**What the index file records about itself** (``meta`` in sqlite, the collection's
|
|
7
|
+
metadata in Chroma):
|
|
8
|
+
|
|
9
|
+
- ``schema``: the layout's version. A file from before it existed (second-brain-joplin
|
|
10
|
+
1.2 and earlier, with ``note_id`` columns) is migrated in place by the sqlite store,
|
|
11
|
+
in one transaction, with no re-embedding; the Chroma store rebuilds it by itself.
|
|
12
|
+
- ``source``: the server that owns it. A second server pointed at the same directory
|
|
13
|
+
is refused: each one's sync would delete the other's items.
|
|
14
|
+
- ``model`` and ``dim``: the embedding model and the vectors' width. Either changing
|
|
15
|
+
means a rebuild (the engine compares them with its embedder).
|
|
16
|
+
|
|
17
|
+
**The word index** (sqlite only) is SQLite FTS5 over the same chunks, with the
|
|
18
|
+
``unicode61 remove_diacritics 2`` tokenizer: case and accents are ignored, and a line
|
|
19
|
+
break is a space. A word hyphenated across a line break (``embed-`` / ``ding``) is
|
|
20
|
+
indexed both as written and joined (``embedding``), so a real hyphen is never lost.
|
|
21
|
+
A SQLite built without FTS5 has no word index; search by meaning still works.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import os
|
|
27
|
+
import re
|
|
28
|
+
import sqlite3
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any, cast
|
|
31
|
+
|
|
32
|
+
from ..errors import (
|
|
33
|
+
IndexOwnedElsewhereError,
|
|
34
|
+
SemanticUnavailableError,
|
|
35
|
+
WordSearchUnavailableError,
|
|
36
|
+
)
|
|
37
|
+
from ..server import Server
|
|
38
|
+
from .types import ChunkRecord, QueryHit, VectorIndex
|
|
39
|
+
|
|
40
|
+
SCHEMA = 2
|
|
41
|
+
STORES = ("sqlite-vec", "chroma")
|
|
42
|
+
_HYPHEN_BREAK = re.compile(r"(\w+)-[ \t]*\r?\n[ \t]*(\w+)")
|
|
43
|
+
_WORD = re.compile(r"\w+")
|
|
44
|
+
_VEC_WIDTH = re.compile(r"float\[(\d+)\]")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def install_hint(server: Server, package: str, extra: str = "semantic") -> str:
|
|
48
|
+
return (
|
|
49
|
+
f"Semantic search needs the optional '{package}' dependency. Install it with "
|
|
50
|
+
f'`pip install "{server.name}[{extra}]"` (or the matching uvx/uv extra).'
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def no_extensions_hint(server: Server) -> str:
|
|
55
|
+
return (
|
|
56
|
+
"The sqlite-vec index needs a Python that can load SQLite extensions, and this "
|
|
57
|
+
"one can't (some macOS and pyenv builds leave it out). Run the server on uv's own "
|
|
58
|
+
"Python (`uv python install 3.13`, then `uvx --python 3.13 ...`), or set "
|
|
59
|
+
f"{server.var('VECTOR_STORE')}=chroma with the [semantic-chroma] extra."
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def owned_elsewhere(server: Server, owner: str) -> IndexOwnedElsewhereError:
|
|
64
|
+
"""The refusal for a directory another server owns. It names the variable, not the
|
|
65
|
+
path: the message can reach a remote client, and a path names the operator's home."""
|
|
66
|
+
return IndexOwnedElsewhereError(
|
|
67
|
+
f"The search index in {server.var('INDEX_DIR')} belongs to {owner}. Give each "
|
|
68
|
+
"server an index directory of its own."
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _private(path: Path, mode: int) -> None:
|
|
73
|
+
"""Make ``path`` readable by its owner only, where the system has modes."""
|
|
74
|
+
if os.name == "posix" and path.exists():
|
|
75
|
+
try:
|
|
76
|
+
path.chmod(mode)
|
|
77
|
+
except OSError: # not ours to change: leave it as it was
|
|
78
|
+
pass
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def joined_words(text: str) -> str:
|
|
82
|
+
"""The words hyphenated across a line break in ``text``, joined.
|
|
83
|
+
|
|
84
|
+
``embed-`` at the end of one line and ``ding`` at the start of the next give
|
|
85
|
+
``embedding``.
|
|
86
|
+
"""
|
|
87
|
+
return " ".join(a + b for a, b in _HYPHEN_BREAK.findall(text))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def match_expression(query: str) -> str | None:
|
|
91
|
+
"""``query`` as an FTS5 expression: every word, each quoted (no operators); None if none."""
|
|
92
|
+
words = _WORD.findall(query)
|
|
93
|
+
return " ".join(f'"{word}"' for word in words) if words else None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def build_vector_index(server: Server, store: str, index_dir: str | Path) -> VectorIndex:
|
|
97
|
+
"""Open (or create) the configured store in ``index_dir``, for ``server``."""
|
|
98
|
+
directory = Path(index_dir)
|
|
99
|
+
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
100
|
+
# An index holds the indexed text: a directory made before this rule (0755) is
|
|
101
|
+
# made private too.
|
|
102
|
+
_private(directory, 0o700)
|
|
103
|
+
if store == "sqlite-vec":
|
|
104
|
+
return SqliteVecIndex(directory, server)
|
|
105
|
+
if store == "chroma":
|
|
106
|
+
return ChromaVectorIndex(directory, server)
|
|
107
|
+
raise SemanticUnavailableError(
|
|
108
|
+
f"Unknown {server.var('VECTOR_STORE')} {store!r}; expected 'sqlite-vec' or 'chroma'."
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class SqliteVecIndex:
|
|
113
|
+
"""One SQLite file: chunk metadata, a vec0 table of vectors, and an FTS5 word index."""
|
|
114
|
+
|
|
115
|
+
def __init__(self, index_dir: Path, server: Server) -> None:
|
|
116
|
+
try:
|
|
117
|
+
import sqlite_vec
|
|
118
|
+
except ImportError as exc: # pragma: no cover - env-dependent
|
|
119
|
+
raise SemanticUnavailableError(install_hint(server, "sqlite-vec")) from exc
|
|
120
|
+
if not hasattr(sqlite3.Connection, "enable_load_extension"):
|
|
121
|
+
# Some Python builds (macOS's framework Python, some pyenv builds) leave
|
|
122
|
+
# out SQLite's extension loading, which sqlite-vec needs.
|
|
123
|
+
raise SemanticUnavailableError(no_extensions_hint(server))
|
|
124
|
+
path = index_dir / "index.db"
|
|
125
|
+
self._conn = sqlite3.connect(str(path), check_same_thread=False)
|
|
126
|
+
try:
|
|
127
|
+
self._conn.enable_load_extension(True)
|
|
128
|
+
sqlite_vec.load(self._conn)
|
|
129
|
+
self._conn.enable_load_extension(False)
|
|
130
|
+
self._serialize = sqlite_vec.serialize_float32
|
|
131
|
+
self._conn.execute("CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT)")
|
|
132
|
+
owner = self._meta("source")
|
|
133
|
+
if owner and owner != server.name:
|
|
134
|
+
raise owned_elsewhere(server, owner)
|
|
135
|
+
self._words = _has_fts5(self._conn)
|
|
136
|
+
if "note_id" in self._columns("chunks"):
|
|
137
|
+
self._migrate_from_1()
|
|
138
|
+
self._create_tables()
|
|
139
|
+
if not owner:
|
|
140
|
+
self._set_meta("source", server.name)
|
|
141
|
+
self._set_meta("schema", str(SCHEMA))
|
|
142
|
+
self._conn.commit()
|
|
143
|
+
except BaseException:
|
|
144
|
+
self._conn.close()
|
|
145
|
+
raise
|
|
146
|
+
_private(path, 0o600)
|
|
147
|
+
|
|
148
|
+
# -- the file's layout -------------------------------------------------------
|
|
149
|
+
|
|
150
|
+
def _columns(self, table: str) -> list[str]:
|
|
151
|
+
return [row[1] for row in self._conn.execute(f"PRAGMA table_info({table})").fetchall()]
|
|
152
|
+
|
|
153
|
+
def _meta(self, key: str) -> str:
|
|
154
|
+
row = self._conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
|
|
155
|
+
return cast(str, row[0]) if row else ""
|
|
156
|
+
|
|
157
|
+
def _set_meta(self, key: str, value: str) -> None:
|
|
158
|
+
self._conn.execute(
|
|
159
|
+
"INSERT INTO meta(key, value) VALUES(?, ?) "
|
|
160
|
+
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
|
161
|
+
(key, value),
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
def _create_tables(self) -> None:
|
|
165
|
+
self._conn.execute(
|
|
166
|
+
"CREATE TABLE IF NOT EXISTS chunks("
|
|
167
|
+
"rowid INTEGER PRIMARY KEY, chunk_id TEXT UNIQUE, item_id TEXT, "
|
|
168
|
+
"version TEXT, title TEXT, chunk_index INTEGER, text TEXT)"
|
|
169
|
+
)
|
|
170
|
+
self._conn.execute(
|
|
171
|
+
"CREATE TABLE IF NOT EXISTS empty_items(item_id TEXT PRIMARY KEY, version TEXT)"
|
|
172
|
+
)
|
|
173
|
+
if self._words:
|
|
174
|
+
self._conn.execute(
|
|
175
|
+
"CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5("
|
|
176
|
+
"text, joined, tokenize = 'unicode61 remove_diacritics 2')"
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
def _migrate_from_1(self) -> None:
|
|
180
|
+
"""A second-brain-joplin 1.2 file: rename to the neutral layout, keep every vector.
|
|
181
|
+
|
|
182
|
+
One transaction: a failure leaves the file exactly as it was.
|
|
183
|
+
"""
|
|
184
|
+
conn = self._conn
|
|
185
|
+
conn.commit()
|
|
186
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
187
|
+
try:
|
|
188
|
+
# Another process may have migrated it while this one waited for the lock.
|
|
189
|
+
if "note_id" not in self._columns("chunks"):
|
|
190
|
+
conn.commit()
|
|
191
|
+
return
|
|
192
|
+
conn.execute("ALTER TABLE chunks RENAME COLUMN note_id TO item_id")
|
|
193
|
+
conn.execute("ALTER TABLE chunks RENAME COLUMN updated_time TO version")
|
|
194
|
+
if self._columns("empty_notes"):
|
|
195
|
+
conn.execute("ALTER TABLE empty_notes RENAME TO empty_items")
|
|
196
|
+
conn.execute("ALTER TABLE empty_items RENAME COLUMN note_id TO item_id")
|
|
197
|
+
conn.execute("ALTER TABLE empty_items RENAME COLUMN updated_time TO version")
|
|
198
|
+
width = self._declared_width()
|
|
199
|
+
if width:
|
|
200
|
+
self._set_meta("dim", str(width))
|
|
201
|
+
self._create_tables()
|
|
202
|
+
if self._words:
|
|
203
|
+
rows = conn.execute("SELECT rowid, text FROM chunks").fetchall()
|
|
204
|
+
conn.executemany(
|
|
205
|
+
"INSERT INTO chunks_fts(rowid, text, joined) VALUES(?, ?, ?)",
|
|
206
|
+
[(rowid, text, joined_words(text)) for rowid, text in rows],
|
|
207
|
+
)
|
|
208
|
+
self._set_meta("schema", str(SCHEMA))
|
|
209
|
+
conn.commit()
|
|
210
|
+
except BaseException:
|
|
211
|
+
conn.rollback()
|
|
212
|
+
raise
|
|
213
|
+
|
|
214
|
+
@property
|
|
215
|
+
def model_name(self) -> str:
|
|
216
|
+
return self._meta("model")
|
|
217
|
+
|
|
218
|
+
@property
|
|
219
|
+
def dim(self) -> int:
|
|
220
|
+
raw = self._meta("dim")
|
|
221
|
+
return int(raw) if raw.isdigit() else 0
|
|
222
|
+
|
|
223
|
+
def _declared_width(self) -> int:
|
|
224
|
+
"""The vectors' width as the vec0 table was declared (0 when there is none)."""
|
|
225
|
+
row = self._conn.execute(
|
|
226
|
+
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'vec_chunks'"
|
|
227
|
+
).fetchone()
|
|
228
|
+
found = _VEC_WIDTH.search(row[0]) if row and row[0] else None
|
|
229
|
+
return int(found.group(1)) if found else 0
|
|
230
|
+
|
|
231
|
+
def _vec_table_ready(self) -> bool:
|
|
232
|
+
row = self._conn.execute(
|
|
233
|
+
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'vec_chunks'"
|
|
234
|
+
).fetchone()
|
|
235
|
+
return row is not None
|
|
236
|
+
|
|
237
|
+
def _ensure_vec_table(self, dim: int) -> None:
|
|
238
|
+
if not self._vec_table_ready():
|
|
239
|
+
self._conn.execute(
|
|
240
|
+
f"CREATE VIRTUAL TABLE vec_chunks USING vec0(embedding float[{dim}])"
|
|
241
|
+
)
|
|
242
|
+
self._set_meta("dim", str(dim))
|
|
243
|
+
self._conn.commit()
|
|
244
|
+
|
|
245
|
+
# -- the protocol ------------------------------------------------------------
|
|
246
|
+
|
|
247
|
+
def item_versions(self) -> dict[str, str]:
|
|
248
|
+
rows = self._conn.execute("SELECT item_id, version FROM chunks").fetchall()
|
|
249
|
+
return {item_id: version for item_id, version in rows}
|
|
250
|
+
|
|
251
|
+
def empty_versions(self) -> dict[str, str]:
|
|
252
|
+
rows = self._conn.execute("SELECT item_id, version FROM empty_items").fetchall()
|
|
253
|
+
return {item_id: version for item_id, version in rows}
|
|
254
|
+
|
|
255
|
+
def mark_empty(self, item_id: str, version: str) -> None:
|
|
256
|
+
self._conn.execute(
|
|
257
|
+
"INSERT INTO empty_items(item_id, version) VALUES(?, ?) "
|
|
258
|
+
"ON CONFLICT(item_id) DO UPDATE SET version = excluded.version",
|
|
259
|
+
(item_id, version),
|
|
260
|
+
)
|
|
261
|
+
self._conn.commit()
|
|
262
|
+
|
|
263
|
+
def upsert(self, records: list[ChunkRecord]) -> None:
|
|
264
|
+
if not records:
|
|
265
|
+
return
|
|
266
|
+
self._ensure_vec_table(len(records[0].embedding))
|
|
267
|
+
item_ids = sorted({record.item_id for record in records})
|
|
268
|
+
self._conn.execute(
|
|
269
|
+
f"DELETE FROM empty_items WHERE item_id IN ({', '.join('?' for _ in item_ids)})",
|
|
270
|
+
item_ids,
|
|
271
|
+
)
|
|
272
|
+
for record in records:
|
|
273
|
+
cursor = self._conn.execute(
|
|
274
|
+
"INSERT INTO chunks(chunk_id, item_id, version, title, chunk_index, text) "
|
|
275
|
+
"VALUES(?, ?, ?, ?, ?, ?)",
|
|
276
|
+
(
|
|
277
|
+
record.chunk_id,
|
|
278
|
+
record.item_id,
|
|
279
|
+
record.version,
|
|
280
|
+
record.title,
|
|
281
|
+
record.chunk_index,
|
|
282
|
+
record.text,
|
|
283
|
+
),
|
|
284
|
+
)
|
|
285
|
+
self._conn.execute(
|
|
286
|
+
"INSERT INTO vec_chunks(rowid, embedding) VALUES(?, ?)",
|
|
287
|
+
(cursor.lastrowid, self._serialize(record.embedding)),
|
|
288
|
+
)
|
|
289
|
+
if self._words:
|
|
290
|
+
self._conn.execute(
|
|
291
|
+
"INSERT INTO chunks_fts(rowid, text, joined) VALUES(?, ?, ?)",
|
|
292
|
+
(cursor.lastrowid, record.text, joined_words(record.text)),
|
|
293
|
+
)
|
|
294
|
+
self._conn.commit()
|
|
295
|
+
|
|
296
|
+
def delete_items(self, item_ids: list[str]) -> None:
|
|
297
|
+
if not item_ids:
|
|
298
|
+
return
|
|
299
|
+
placeholders = ", ".join("?" for _ in item_ids)
|
|
300
|
+
rows = self._conn.execute(
|
|
301
|
+
f"SELECT rowid FROM chunks WHERE item_id IN ({placeholders})", item_ids
|
|
302
|
+
).fetchall()
|
|
303
|
+
rowids = [r[0] for r in rows]
|
|
304
|
+
self._conn.execute(f"DELETE FROM chunks WHERE item_id IN ({placeholders})", item_ids)
|
|
305
|
+
self._conn.execute(f"DELETE FROM empty_items WHERE item_id IN ({placeholders})", item_ids)
|
|
306
|
+
if rowids:
|
|
307
|
+
rowid_placeholders = ", ".join("?" for _ in rowids)
|
|
308
|
+
if self._vec_table_ready():
|
|
309
|
+
self._conn.execute(
|
|
310
|
+
f"DELETE FROM vec_chunks WHERE rowid IN ({rowid_placeholders})", rowids
|
|
311
|
+
)
|
|
312
|
+
if self._words:
|
|
313
|
+
self._conn.execute(
|
|
314
|
+
f"DELETE FROM chunks_fts WHERE rowid IN ({rowid_placeholders})", rowids
|
|
315
|
+
)
|
|
316
|
+
self._conn.commit()
|
|
317
|
+
|
|
318
|
+
def query(self, embedding: list[float], k: int) -> list[QueryHit]:
|
|
319
|
+
if not self._vec_table_ready():
|
|
320
|
+
return []
|
|
321
|
+
rows = self._conn.execute(
|
|
322
|
+
"SELECT c.item_id, c.title, c.text, c.chunk_index, v.distance "
|
|
323
|
+
"FROM vec_chunks v JOIN chunks c ON c.rowid = v.rowid "
|
|
324
|
+
"WHERE v.embedding MATCH ? AND k = ? ORDER BY v.distance",
|
|
325
|
+
(self._serialize(embedding), k),
|
|
326
|
+
).fetchall()
|
|
327
|
+
return [
|
|
328
|
+
QueryHit(
|
|
329
|
+
item_id=item_id,
|
|
330
|
+
title=title,
|
|
331
|
+
text=text,
|
|
332
|
+
chunk_index=chunk_index,
|
|
333
|
+
score=1.0 / (1.0 + float(distance)),
|
|
334
|
+
)
|
|
335
|
+
for item_id, title, text, chunk_index, distance in rows
|
|
336
|
+
]
|
|
337
|
+
|
|
338
|
+
def word_search(self, query: str, k: int) -> list[QueryHit]:
|
|
339
|
+
if not self._words:
|
|
340
|
+
raise WordSearchUnavailableError(
|
|
341
|
+
"This SQLite has no FTS5, so the index has no word search. "
|
|
342
|
+
"Search by meaning still works."
|
|
343
|
+
)
|
|
344
|
+
expression = match_expression(query)
|
|
345
|
+
if expression is None:
|
|
346
|
+
return []
|
|
347
|
+
rows = self._conn.execute(
|
|
348
|
+
"SELECT c.item_id, c.title, c.text, c.chunk_index, f.rank "
|
|
349
|
+
"FROM chunks_fts f JOIN chunks c ON c.rowid = f.rowid "
|
|
350
|
+
"WHERE chunks_fts MATCH ? ORDER BY f.rank LIMIT ?",
|
|
351
|
+
(expression, k),
|
|
352
|
+
).fetchall()
|
|
353
|
+
# FTS5's rank is bm25: negative, and lower for a better match.
|
|
354
|
+
return [
|
|
355
|
+
QueryHit(
|
|
356
|
+
item_id=item_id,
|
|
357
|
+
title=title,
|
|
358
|
+
text=text,
|
|
359
|
+
chunk_index=chunk_index,
|
|
360
|
+
score=-float(rank),
|
|
361
|
+
)
|
|
362
|
+
for item_id, title, text, chunk_index, rank in rows
|
|
363
|
+
]
|
|
364
|
+
|
|
365
|
+
def reset(self, model_name: str) -> None:
|
|
366
|
+
# One transaction: an interrupted rebuild must not leave chunks without vectors
|
|
367
|
+
# (their versions would say there is nothing to embed).
|
|
368
|
+
conn = self._conn
|
|
369
|
+
conn.commit()
|
|
370
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
371
|
+
try:
|
|
372
|
+
conn.execute("DROP TABLE IF EXISTS vec_chunks")
|
|
373
|
+
conn.execute("DELETE FROM chunks")
|
|
374
|
+
conn.execute("DELETE FROM empty_items")
|
|
375
|
+
if self._words:
|
|
376
|
+
conn.execute("DELETE FROM chunks_fts")
|
|
377
|
+
conn.execute("DELETE FROM meta WHERE key = 'dim'")
|
|
378
|
+
self._set_meta("model", model_name)
|
|
379
|
+
conn.commit()
|
|
380
|
+
except BaseException:
|
|
381
|
+
conn.rollback()
|
|
382
|
+
raise
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _has_fts5(conn: sqlite3.Connection) -> bool:
|
|
386
|
+
try:
|
|
387
|
+
conn.execute("CREATE VIRTUAL TABLE temp.fts5_probe USING fts5(x)")
|
|
388
|
+
conn.execute("DROP TABLE temp.fts5_probe")
|
|
389
|
+
except sqlite3.OperationalError:
|
|
390
|
+
return False
|
|
391
|
+
return True
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
class ChromaVectorIndex:
|
|
395
|
+
"""A persistent Chroma collection of chunks, keyed by chunk id. No word search."""
|
|
396
|
+
|
|
397
|
+
_COLLECTION = "items"
|
|
398
|
+
# Items with no text. Chroma needs a vector on every record, so each mark carries a
|
|
399
|
+
# one-number placeholder; it lives in its own collection, which no search reads.
|
|
400
|
+
_EMPTY_COLLECTION = "empty_items"
|
|
401
|
+
# A second-brain-joplin 1.2 index: rebuilt by itself (dropped here, then the
|
|
402
|
+
# engine finds no model and embeds everything again).
|
|
403
|
+
_OLD_COLLECTIONS = ("notes", "empty_notes")
|
|
404
|
+
_PLACEHOLDER = [1.0]
|
|
405
|
+
|
|
406
|
+
def __init__(self, index_dir: Path, server: Server) -> None:
|
|
407
|
+
try:
|
|
408
|
+
import chromadb
|
|
409
|
+
except ImportError as exc: # pragma: no cover - env-dependent
|
|
410
|
+
raise SemanticUnavailableError(
|
|
411
|
+
install_hint(server, "chromadb", extra="semantic-chroma")
|
|
412
|
+
) from exc
|
|
413
|
+
self._server = server
|
|
414
|
+
self._client = chromadb.PersistentClient(path=str(index_dir / "chroma"))
|
|
415
|
+
existing = {c if isinstance(c, str) else c.name for c in self._client.list_collections()}
|
|
416
|
+
if self._COLLECTION in existing:
|
|
417
|
+
owner = (self._client.get_collection(self._COLLECTION).metadata or {}).get("source")
|
|
418
|
+
if owner and owner != server.name:
|
|
419
|
+
raise owned_elsewhere(server, str(owner))
|
|
420
|
+
for name in self._OLD_COLLECTIONS:
|
|
421
|
+
if name in existing:
|
|
422
|
+
self._client.delete_collection(name)
|
|
423
|
+
if self._COLLECTION in existing:
|
|
424
|
+
# Opened without metadata, then only missing keys added: some chromadb
|
|
425
|
+
# versions overwrite an existing collection's metadata with what a
|
|
426
|
+
# get-or-create passes, which would drop the model and width (a rebuild
|
|
427
|
+
# at every start).
|
|
428
|
+
self._collection = self._client.get_collection(self._COLLECTION)
|
|
429
|
+
current = dict(self._collection.metadata or {})
|
|
430
|
+
missing = {k: v for k, v in self._new_metadata("").items() if k not in current}
|
|
431
|
+
if missing:
|
|
432
|
+
self._collection.modify(metadata={**current, **missing})
|
|
433
|
+
self._empty = self._client.get_or_create_collection(self._EMPTY_COLLECTION)
|
|
434
|
+
else:
|
|
435
|
+
self._create("")
|
|
436
|
+
|
|
437
|
+
def _create(self, model_name: str) -> None:
|
|
438
|
+
"""Make the collections afresh (after deleting them, or for a new index)."""
|
|
439
|
+
self._collection = self._client.create_collection(
|
|
440
|
+
self._COLLECTION, metadata=self._new_metadata(model_name)
|
|
441
|
+
)
|
|
442
|
+
self._empty = self._client.get_or_create_collection(self._EMPTY_COLLECTION)
|
|
443
|
+
|
|
444
|
+
def _new_metadata(self, model_name: str) -> dict[str, Any]:
|
|
445
|
+
metadata: dict[str, Any] = {"schema": SCHEMA, "source": self._server.name}
|
|
446
|
+
if model_name:
|
|
447
|
+
metadata["model"] = model_name
|
|
448
|
+
return metadata
|
|
449
|
+
|
|
450
|
+
@property
|
|
451
|
+
def model_name(self) -> str:
|
|
452
|
+
return cast(str, (self._collection.metadata or {}).get("model", ""))
|
|
453
|
+
|
|
454
|
+
@property
|
|
455
|
+
def dim(self) -> int:
|
|
456
|
+
return int((self._collection.metadata or {}).get("dim", 0))
|
|
457
|
+
|
|
458
|
+
def item_versions(self) -> dict[str, str]:
|
|
459
|
+
data = self._collection.get(include=["metadatas"])
|
|
460
|
+
return {
|
|
461
|
+
str(metadata["item_id"]): str(metadata["version"])
|
|
462
|
+
for metadata in data.get("metadatas") or []
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
def empty_versions(self) -> dict[str, str]:
|
|
466
|
+
data = self._empty.get(include=["metadatas"])
|
|
467
|
+
ids = data.get("ids") or []
|
|
468
|
+
metadatas = data.get("metadatas") or []
|
|
469
|
+
return {
|
|
470
|
+
str(item_id): str(metadata["version"])
|
|
471
|
+
for item_id, metadata in zip(ids, metadatas, strict=True)
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
def mark_empty(self, item_id: str, version: str) -> None:
|
|
475
|
+
self._empty.upsert(
|
|
476
|
+
ids=[item_id], embeddings=[self._PLACEHOLDER], metadatas=[{"version": version}]
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
def upsert(self, records: list[ChunkRecord]) -> None:
|
|
480
|
+
if not records:
|
|
481
|
+
return
|
|
482
|
+
if not self.dim:
|
|
483
|
+
metadata = dict(self._collection.metadata or {})
|
|
484
|
+
metadata["dim"] = len(records[0].embedding)
|
|
485
|
+
self._collection.modify(metadata=metadata)
|
|
486
|
+
self._empty.delete(ids=sorted({r.item_id for r in records}))
|
|
487
|
+
self._collection.upsert(
|
|
488
|
+
ids=[r.chunk_id for r in records],
|
|
489
|
+
embeddings=[r.embedding for r in records],
|
|
490
|
+
documents=[r.text for r in records],
|
|
491
|
+
metadatas=[
|
|
492
|
+
{
|
|
493
|
+
"item_id": r.item_id,
|
|
494
|
+
"title": r.title,
|
|
495
|
+
"version": r.version,
|
|
496
|
+
"chunk_index": r.chunk_index,
|
|
497
|
+
}
|
|
498
|
+
for r in records
|
|
499
|
+
],
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
def delete_items(self, item_ids: list[str]) -> None:
|
|
503
|
+
if not item_ids:
|
|
504
|
+
return
|
|
505
|
+
self._collection.delete(where={"item_id": {"$in": item_ids}})
|
|
506
|
+
self._empty.delete(ids=item_ids)
|
|
507
|
+
|
|
508
|
+
def query(self, embedding: list[float], k: int) -> list[QueryHit]:
|
|
509
|
+
result: dict[str, Any] = self._collection.query(
|
|
510
|
+
query_embeddings=[embedding],
|
|
511
|
+
n_results=k,
|
|
512
|
+
include=["metadatas", "documents", "distances"],
|
|
513
|
+
)
|
|
514
|
+
metadatas = (result.get("metadatas") or [[]])[0]
|
|
515
|
+
documents = (result.get("documents") or [[]])[0]
|
|
516
|
+
distances = (result.get("distances") or [[]])[0]
|
|
517
|
+
return [
|
|
518
|
+
QueryHit(
|
|
519
|
+
item_id=str(metadata["item_id"]),
|
|
520
|
+
title=str(metadata["title"]),
|
|
521
|
+
text=str(document),
|
|
522
|
+
chunk_index=int(metadata["chunk_index"]),
|
|
523
|
+
score=1.0 / (1.0 + float(distance)),
|
|
524
|
+
)
|
|
525
|
+
for metadata, document, distance in zip(metadatas, documents, distances)
|
|
526
|
+
]
|
|
527
|
+
|
|
528
|
+
def word_search(self, query: str, k: int) -> list[QueryHit]:
|
|
529
|
+
raise WordSearchUnavailableError(
|
|
530
|
+
"The Chroma store has no word search; the default store (sqlite-vec) has."
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
def reset(self, model_name: str) -> None:
|
|
534
|
+
for name in (self._COLLECTION, self._EMPTY_COLLECTION):
|
|
535
|
+
try:
|
|
536
|
+
self._client.delete_collection(name)
|
|
537
|
+
except Exception: # noqa: BLE001 - the collection may not exist yet
|
|
538
|
+
pass
|
|
539
|
+
self._create(model_name)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Pure comparison of a source's current items with the indexed versions."""
|
|
2
|
+
|
|
3
|
+
from .types import SyncPlan
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def diff_versions(current: dict[str, str], indexed: dict[str, str]) -> SyncPlan:
|
|
7
|
+
"""Which items need (re)embedding and which to drop.
|
|
8
|
+
|
|
9
|
+
``current`` maps item id → version marker for every visible item; ``indexed``
|
|
10
|
+
is the same map for what the index holds. An item is *changed* when it is new or
|
|
11
|
+
its marker differs; *deleted* when it is in the index but no longer visible.
|
|
12
|
+
"""
|
|
13
|
+
changed_ids = [
|
|
14
|
+
item_id for item_id, version in current.items() if indexed.get(item_id) != version
|
|
15
|
+
]
|
|
16
|
+
deleted_ids = [item_id for item_id in indexed if item_id not in current]
|
|
17
|
+
return SyncPlan(changed_ids=changed_ids, deleted_ids=deleted_ids)
|