second-brain-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- second_brain_core/__init__.py +32 -0
- second_brain_core/annotations.py +15 -0
- second_brain_core/audit.py +275 -0
- second_brain_core/auth.py +235 -0
- second_brain_core/env.py +171 -0
- second_brain_core/errors.py +37 -0
- second_brain_core/failure.py +87 -0
- second_brain_core/health.py +48 -0
- second_brain_core/py.typed +0 -0
- second_brain_core/recent.py +69 -0
- second_brain_core/remote.py +252 -0
- second_brain_core/semantic/__init__.py +7 -0
- second_brain_core/semantic/chunking.py +48 -0
- second_brain_core/semantic/embedders.py +68 -0
- second_brain_core/semantic/engine.py +218 -0
- second_brain_core/semantic/stores.py +539 -0
- second_brain_core/semantic/sync.py +17 -0
- second_brain_core/semantic/testing.py +183 -0
- second_brain_core/semantic/types.py +130 -0
- second_brain_core/server.py +44 -0
- second_brain_core/testing.py +57 -0
- second_brain_core-0.1.0.dist-info/METADATA +106 -0
- second_brain_core-0.1.0.dist-info/RECORD +25 -0
- second_brain_core-0.1.0.dist-info/WHEEL +4 -0
- second_brain_core-0.1.0.dist-info/licenses/LICENSE +661 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Split an item's text into overlapping chunks for embedding.
|
|
2
|
+
|
|
3
|
+
Pure and deterministic: paragraph-aware packing into fixed-size character
|
|
4
|
+
windows with a small overlap, each chunk prefixed with the item's title so the
|
|
5
|
+
embedding carries a bit of context.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
CHUNK_SIZE = 1000
|
|
11
|
+
CHUNK_OVERLAP = 100
|
|
12
|
+
|
|
13
|
+
_PARAGRAPH_SPLIT = re.compile(r"\n\s*\n")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def chunk_text(
|
|
17
|
+
title: str,
|
|
18
|
+
body: str,
|
|
19
|
+
*,
|
|
20
|
+
chunk_size: int = CHUNK_SIZE,
|
|
21
|
+
overlap: int = CHUNK_OVERLAP,
|
|
22
|
+
) -> list[str]:
|
|
23
|
+
"""Return title-prefixed chunks of ``body``; empty/whitespace body → ``[]``."""
|
|
24
|
+
text = body.strip()
|
|
25
|
+
if not text:
|
|
26
|
+
return []
|
|
27
|
+
|
|
28
|
+
paragraphs = [p.strip() for p in _PARAGRAPH_SPLIT.split(text) if p.strip()]
|
|
29
|
+
|
|
30
|
+
chunks: list[str] = []
|
|
31
|
+
current = ""
|
|
32
|
+
for para in paragraphs:
|
|
33
|
+
if not current:
|
|
34
|
+
current = para
|
|
35
|
+
elif len(current) + 2 + len(para) <= chunk_size:
|
|
36
|
+
current = f"{current}\n\n{para}"
|
|
37
|
+
else:
|
|
38
|
+
chunks.append(current)
|
|
39
|
+
current = para
|
|
40
|
+
while len(current) > chunk_size:
|
|
41
|
+
chunks.append(current[:chunk_size])
|
|
42
|
+
current = current[chunk_size - overlap :]
|
|
43
|
+
# `current` is always non-empty here: text was non-empty, so there is at least
|
|
44
|
+
# one paragraph, and each window split leaves a non-empty remainder.
|
|
45
|
+
chunks.append(current)
|
|
46
|
+
|
|
47
|
+
prefix = f"{title.strip()}\n\n" if title.strip() else ""
|
|
48
|
+
return [f"{prefix}{chunk}" for chunk in chunks]
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""The embedders, backed by the optional extras.
|
|
2
|
+
|
|
3
|
+
Coverage-omitted: they need a model download, so CI doesn't run them. They are
|
|
4
|
+
type-checked by mypy (the optional libraries resolve to ``Any``).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from ..errors import SemanticUnavailableError
|
|
10
|
+
from ..server import Server
|
|
11
|
+
from .stores import install_hint
|
|
12
|
+
from .types import Embedder
|
|
13
|
+
|
|
14
|
+
BACKENDS = ("fastembed", "sentence-transformers")
|
|
15
|
+
FASTEMBED_DEFAULT = "BAAI/bge-small-en-v1.5"
|
|
16
|
+
SENTENCE_TRANSFORMERS_DEFAULT = "BAAI/bge-m3"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def build_embedder(server: Server, backend: str, model: str = "") -> Embedder:
|
|
20
|
+
"""The configured embedder (``model`` empty: the backend's default), or a clear refusal."""
|
|
21
|
+
if backend == "fastembed":
|
|
22
|
+
return _FastEmbedEmbedder(server, model or FASTEMBED_DEFAULT)
|
|
23
|
+
if backend == "sentence-transformers":
|
|
24
|
+
return _SentenceTransformerEmbedder(server, model or SENTENCE_TRANSFORMERS_DEFAULT)
|
|
25
|
+
raise SemanticUnavailableError(
|
|
26
|
+
f"Unknown {server.var('EMBEDDING_BACKEND')} {backend!r}; expected 'fastembed' or "
|
|
27
|
+
"'sentence-transformers'."
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _FastEmbedEmbedder:
|
|
32
|
+
def __init__(self, server: Server, model_name: str) -> None:
|
|
33
|
+
try:
|
|
34
|
+
from fastembed import TextEmbedding
|
|
35
|
+
except ImportError as exc: # pragma: no cover - env-dependent
|
|
36
|
+
raise SemanticUnavailableError(install_hint(server, "fastembed")) from exc
|
|
37
|
+
self.model_name = model_name
|
|
38
|
+
self._model = TextEmbedding(model_name=model_name)
|
|
39
|
+
probe = next(iter(self._model.embed(["probe"])))
|
|
40
|
+
self.dim = len(probe)
|
|
41
|
+
|
|
42
|
+
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
|
43
|
+
return [[float(x) for x in vec] for vec in self._model.embed(texts)]
|
|
44
|
+
|
|
45
|
+
def embed_query(self, text: str) -> list[float]:
|
|
46
|
+
vec = next(iter(self._model.query_embed([text])))
|
|
47
|
+
return [float(x) for x in vec]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class _SentenceTransformerEmbedder:
|
|
51
|
+
def __init__(self, server: Server, model_name: str) -> None:
|
|
52
|
+
try:
|
|
53
|
+
from sentence_transformers import SentenceTransformer
|
|
54
|
+
except ImportError as exc: # pragma: no cover - env-dependent
|
|
55
|
+
raise SemanticUnavailableError(
|
|
56
|
+
install_hint(server, "sentence-transformers", extra="semantic-st")
|
|
57
|
+
) from exc
|
|
58
|
+
self.model_name = model_name
|
|
59
|
+
self._model = SentenceTransformer(model_name)
|
|
60
|
+
self.dim = int(self._model.get_sentence_embedding_dimension())
|
|
61
|
+
|
|
62
|
+
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
|
63
|
+
vecs = self._model.encode(texts, normalize_embeddings=True)
|
|
64
|
+
return [[float(x) for x in vec] for vec in vecs]
|
|
65
|
+
|
|
66
|
+
def embed_query(self, text: str) -> list[float]:
|
|
67
|
+
vec = self._model.encode([text], normalize_embeddings=True)[0]
|
|
68
|
+
return [float(x) for x in vec]
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""Keeps a server's index in step with its source, and searches it.
|
|
2
|
+
|
|
3
|
+
Pure orchestration: it takes a :class:`~.types.Source`, an :class:`~.types.Embedder`
|
|
4
|
+
and a :class:`~.types.VectorIndex` by injection and never imports an optional
|
|
5
|
+
dependency. Embedder and store calls run on worker threads, a lock keeps two syncs
|
|
6
|
+
from writing the store at once, and a semaphore caps the searches that embed and
|
|
7
|
+
query together.
|
|
8
|
+
|
|
9
|
+
The source decides what the index may hold: an item it hides is never embedded,
|
|
10
|
+
and one that becomes hidden is dropped by the next sync, before anything new is
|
|
11
|
+
embedded.
|
|
12
|
+
|
|
13
|
+
What a search returns does not rest on the index being clean, though. The index is
|
|
14
|
+
a file on disk that other processes can write, each with settings of its own (a
|
|
15
|
+
reindex run without the server's env file may have no filter and embed
|
|
16
|
+
everything). So with a filter active, every search asks the source which items are
|
|
17
|
+
visible now and returns only those, whatever the index holds.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
from enum import Enum
|
|
22
|
+
|
|
23
|
+
from .chunking import chunk_text
|
|
24
|
+
from .sync import diff_versions
|
|
25
|
+
from .types import ChunkRecord, Embedder, ItemText, QueryHit, Skip, Source, SyncStats, VectorIndex
|
|
26
|
+
|
|
27
|
+
# Over-fetch factor, so keeping one hit per item still yields up to `limit` items.
|
|
28
|
+
QUERY_OVERFETCH = 4
|
|
29
|
+
# Searches that may embed and query at the same time.
|
|
30
|
+
SEARCH_CONCURRENCY = 2
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class _Embedded(Enum):
|
|
34
|
+
"""What became of one item a sync looked at."""
|
|
35
|
+
|
|
36
|
+
TEXT = "text" # its chunks are in the index
|
|
37
|
+
EMPTY = "empty" # it has no text: marked, so the next sync leaves it alone
|
|
38
|
+
HIDDEN = "hidden" # the source hides it now: nothing kept
|
|
39
|
+
NOT_READY = "not-ready" # not readable yet: not marked, so the next sync looks again
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Engine:
|
|
43
|
+
"""Keeps the index in step with the source and answers searches."""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
source: Source,
|
|
48
|
+
embedder: Embedder,
|
|
49
|
+
index: VectorIndex,
|
|
50
|
+
*,
|
|
51
|
+
auto_sync: bool = True,
|
|
52
|
+
) -> None:
|
|
53
|
+
self._source = source
|
|
54
|
+
self._embedder = embedder
|
|
55
|
+
self._index = index
|
|
56
|
+
self._auto_sync = auto_sync
|
|
57
|
+
self._lock = asyncio.Lock()
|
|
58
|
+
# Query embedding runs on worker threads; a remote caller firing many
|
|
59
|
+
# searches at once could otherwise tie up the model and the CPU.
|
|
60
|
+
self._search_slots = asyncio.Semaphore(SEARCH_CONCURRENCY)
|
|
61
|
+
|
|
62
|
+
async def sync(self, *, full: bool = False) -> SyncStats:
|
|
63
|
+
"""Re-embed changed items (or rebuild everything)."""
|
|
64
|
+
async with self._lock:
|
|
65
|
+
stats, _ = await self._sync_locked(full=full)
|
|
66
|
+
return stats
|
|
67
|
+
|
|
68
|
+
async def prune(self) -> int:
|
|
69
|
+
"""Drop indexed items that are hidden or gone, embedding nothing.
|
|
70
|
+
|
|
71
|
+
The cheap half of a sync (one listing, no model). Returns how many items
|
|
72
|
+
with text left the index.
|
|
73
|
+
"""
|
|
74
|
+
current = await self._source.list_versions()
|
|
75
|
+
async with self._lock:
|
|
76
|
+
return await self._drop_missing(current)
|
|
77
|
+
|
|
78
|
+
async def _drop_missing(self, current: dict[str, str]) -> int:
|
|
79
|
+
"""Remove every item that isn't in ``current``. Call with the lock."""
|
|
80
|
+
indexed = await asyncio.to_thread(self._index.item_versions)
|
|
81
|
+
empty = await asyncio.to_thread(self._index.empty_versions)
|
|
82
|
+
gone = [item_id for item_id in {**empty, **indexed} if item_id not in current]
|
|
83
|
+
if gone:
|
|
84
|
+
await asyncio.to_thread(self._index.delete_items, gone)
|
|
85
|
+
return sum(1 for item_id in gone if item_id in indexed)
|
|
86
|
+
|
|
87
|
+
def _needs_rebuild(self) -> bool:
|
|
88
|
+
"""The stored vectors come from another model, or have another width."""
|
|
89
|
+
if self._index.model_name != self._embedder.model_name:
|
|
90
|
+
return True
|
|
91
|
+
return bool(self._index.dim) and self._index.dim != self._embedder.dim
|
|
92
|
+
|
|
93
|
+
async def _sync_locked(self, *, full: bool) -> tuple[SyncStats, set[str]]:
|
|
94
|
+
"""Sync, and say which items were visible when it looked. Call with the lock."""
|
|
95
|
+
# Ask the source first: if it is down, or its filter doesn't resolve, the
|
|
96
|
+
# index is left exactly as it was (a full rebuild included).
|
|
97
|
+
current = await self._source.list_versions()
|
|
98
|
+
|
|
99
|
+
model = self._embedder.model_name
|
|
100
|
+
if full or self._needs_rebuild():
|
|
101
|
+
await asyncio.to_thread(self._index.reset, model)
|
|
102
|
+
indexed: dict[str, str] = {}
|
|
103
|
+
empty: dict[str, str] = {}
|
|
104
|
+
else:
|
|
105
|
+
indexed = await asyncio.to_thread(self._index.item_versions)
|
|
106
|
+
empty = await asyncio.to_thread(self._index.empty_versions)
|
|
107
|
+
|
|
108
|
+
# An item with no text was looked at once for its version and marked, so it
|
|
109
|
+
# isn't read again until it changes.
|
|
110
|
+
plan = diff_versions(current, {**empty, **indexed})
|
|
111
|
+
|
|
112
|
+
# Drop before embedding: an item that is now hidden must leave the index even
|
|
113
|
+
# if an embedding below fails.
|
|
114
|
+
if plan.deleted_ids:
|
|
115
|
+
await asyncio.to_thread(self._index.delete_items, plan.deleted_ids)
|
|
116
|
+
|
|
117
|
+
added = 0
|
|
118
|
+
updated = 0
|
|
119
|
+
for item_id in plan.changed_ids:
|
|
120
|
+
outcome = await self._embed_item(item_id, current[item_id])
|
|
121
|
+
if item_id in indexed:
|
|
122
|
+
# Its text changed, or it was emptied: either way its entry did.
|
|
123
|
+
if outcome in (_Embedded.TEXT, _Embedded.EMPTY):
|
|
124
|
+
updated += 1
|
|
125
|
+
elif outcome is _Embedded.TEXT: # new, or an empty item gained text
|
|
126
|
+
added += 1
|
|
127
|
+
|
|
128
|
+
stats = SyncStats(
|
|
129
|
+
added=added,
|
|
130
|
+
updated=updated,
|
|
131
|
+
deleted=sum(1 for item_id in plan.deleted_ids if item_id in indexed),
|
|
132
|
+
total=len(current),
|
|
133
|
+
model_name=model,
|
|
134
|
+
)
|
|
135
|
+
return stats, set(current)
|
|
136
|
+
|
|
137
|
+
async def _embed_item(self, item_id: str, version: str) -> _Embedded:
|
|
138
|
+
"""(Re)embed one item, or mark it as having no text, or leave it out."""
|
|
139
|
+
item = await self._source.get_item(item_id)
|
|
140
|
+
|
|
141
|
+
# Drop any stale chunks (and mark) first, so an item that shrank leaves
|
|
142
|
+
# nothing behind.
|
|
143
|
+
await asyncio.to_thread(self._index.delete_items, [item_id])
|
|
144
|
+
if item is Skip.HIDDEN:
|
|
145
|
+
return _Embedded.HIDDEN
|
|
146
|
+
if item is Skip.NOT_READY:
|
|
147
|
+
return _Embedded.NOT_READY
|
|
148
|
+
assert isinstance(item, ItemText)
|
|
149
|
+
|
|
150
|
+
chunks = chunk_text(item.title, item.text)
|
|
151
|
+
if not chunks:
|
|
152
|
+
await asyncio.to_thread(self._index.mark_empty, item_id, version)
|
|
153
|
+
return _Embedded.EMPTY
|
|
154
|
+
|
|
155
|
+
embeddings = await asyncio.to_thread(self._embedder.embed_documents, chunks)
|
|
156
|
+
records = [
|
|
157
|
+
ChunkRecord(
|
|
158
|
+
chunk_id=f"{item_id}::{i}",
|
|
159
|
+
item_id=item_id,
|
|
160
|
+
title=item.title,
|
|
161
|
+
version=version,
|
|
162
|
+
chunk_index=i,
|
|
163
|
+
text=text,
|
|
164
|
+
embedding=embedding,
|
|
165
|
+
)
|
|
166
|
+
for i, (text, embedding) in enumerate(zip(chunks, embeddings, strict=True))
|
|
167
|
+
]
|
|
168
|
+
await asyncio.to_thread(self._index.upsert, records)
|
|
169
|
+
return _Embedded.TEXT
|
|
170
|
+
|
|
171
|
+
async def _visible_for_search(self) -> set[str] | None:
|
|
172
|
+
"""Bring the index up to date for a search; the visible ids, or None for all.
|
|
173
|
+
|
|
174
|
+
With auto-sync, a sync. Without it, nothing, unless a filter is active: the
|
|
175
|
+
filter still has to hold now, so the source is asked which items are
|
|
176
|
+
visible, and the index is tidied if no sync is using it (a running sync
|
|
177
|
+
tidies it anyway, and a search shouldn't wait behind a long one).
|
|
178
|
+
"""
|
|
179
|
+
if self._auto_sync:
|
|
180
|
+
async with self._lock:
|
|
181
|
+
_, visible = await self._sync_locked(full=False)
|
|
182
|
+
return visible if self._source.filter_active else None
|
|
183
|
+
if not self._source.filter_active:
|
|
184
|
+
return None
|
|
185
|
+
current = await self._source.list_versions()
|
|
186
|
+
if not self._lock.locked():
|
|
187
|
+
async with self._lock:
|
|
188
|
+
await self._drop_missing(current)
|
|
189
|
+
return set(current)
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def _best(hits: list[QueryHit], visible: set[str] | None, limit: int) -> list[QueryHit]:
|
|
193
|
+
# Trust the source's answer, not the index's contents: another process may
|
|
194
|
+
# have written hidden items into the same index since it was tidied.
|
|
195
|
+
if visible is not None:
|
|
196
|
+
hits = [hit for hit in hits if hit.item_id in visible]
|
|
197
|
+
best: dict[str, QueryHit] = {}
|
|
198
|
+
for hit in hits: # hits are ordered best-first
|
|
199
|
+
best.setdefault(hit.item_id, hit)
|
|
200
|
+
return list(best.values())[:limit]
|
|
201
|
+
|
|
202
|
+
async def search(self, query: str, limit: int = 10) -> list[QueryHit]:
|
|
203
|
+
"""Up to ``limit`` items closest in meaning to ``query``: each one's best chunk."""
|
|
204
|
+
visible = await self._visible_for_search()
|
|
205
|
+
async with self._search_slots:
|
|
206
|
+
embedding = await asyncio.to_thread(self._embedder.embed_query, query)
|
|
207
|
+
hits = await asyncio.to_thread(self._index.query, embedding, limit * QUERY_OVERFETCH)
|
|
208
|
+
return self._best(hits, visible, limit)
|
|
209
|
+
|
|
210
|
+
async def word_search(self, query: str, limit: int = 10) -> list[QueryHit]:
|
|
211
|
+
"""Up to ``limit`` items with every word of ``query``, ignoring case and accents.
|
|
212
|
+
|
|
213
|
+
Raises :class:`~second_brain_core.errors.WordSearchUnavailableError` when the
|
|
214
|
+
store has no word index.
|
|
215
|
+
"""
|
|
216
|
+
visible = await self._visible_for_search()
|
|
217
|
+
hits = await asyncio.to_thread(self._index.word_search, query, limit * QUERY_OVERFETCH)
|
|
218
|
+
return self._best(hits, visible, limit)
|