second-brain-core 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,48 @@
1
+ """Split an item's text into overlapping chunks for embedding.
2
+
3
+ Pure and deterministic: paragraph-aware packing into fixed-size character
4
+ windows with a small overlap, each chunk prefixed with the item's title so the
5
+ embedding carries a bit of context.
6
+ """
7
+
8
+ import re
9
+
10
+ CHUNK_SIZE = 1000
11
+ CHUNK_OVERLAP = 100
12
+
13
+ _PARAGRAPH_SPLIT = re.compile(r"\n\s*\n")
14
+
15
+
16
+ def chunk_text(
17
+ title: str,
18
+ body: str,
19
+ *,
20
+ chunk_size: int = CHUNK_SIZE,
21
+ overlap: int = CHUNK_OVERLAP,
22
+ ) -> list[str]:
23
+ """Return title-prefixed chunks of ``body``; empty/whitespace body → ``[]``."""
24
+ text = body.strip()
25
+ if not text:
26
+ return []
27
+
28
+ paragraphs = [p.strip() for p in _PARAGRAPH_SPLIT.split(text) if p.strip()]
29
+
30
+ chunks: list[str] = []
31
+ current = ""
32
+ for para in paragraphs:
33
+ if not current:
34
+ current = para
35
+ elif len(current) + 2 + len(para) <= chunk_size:
36
+ current = f"{current}\n\n{para}"
37
+ else:
38
+ chunks.append(current)
39
+ current = para
40
+ while len(current) > chunk_size:
41
+ chunks.append(current[:chunk_size])
42
+ current = current[chunk_size - overlap :]
43
+ # `current` is always non-empty here: text was non-empty, so there is at least
44
+ # one paragraph, and each window split leaves a non-empty remainder.
45
+ chunks.append(current)
46
+
47
+ prefix = f"{title.strip()}\n\n" if title.strip() else ""
48
+ return [f"{prefix}{chunk}" for chunk in chunks]
@@ -0,0 +1,68 @@
1
+ """The embedders, backed by the optional extras.
2
+
3
+ Coverage-omitted: they need a model download, so CI doesn't run them. They are
4
+ type-checked by mypy (the optional libraries resolve to ``Any``).
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from ..errors import SemanticUnavailableError
10
+ from ..server import Server
11
+ from .stores import install_hint
12
+ from .types import Embedder
13
+
14
+ BACKENDS = ("fastembed", "sentence-transformers")
15
+ FASTEMBED_DEFAULT = "BAAI/bge-small-en-v1.5"
16
+ SENTENCE_TRANSFORMERS_DEFAULT = "BAAI/bge-m3"
17
+
18
+
19
+ def build_embedder(server: Server, backend: str, model: str = "") -> Embedder:
20
+ """The configured embedder (``model`` empty: the backend's default), or a clear refusal."""
21
+ if backend == "fastembed":
22
+ return _FastEmbedEmbedder(server, model or FASTEMBED_DEFAULT)
23
+ if backend == "sentence-transformers":
24
+ return _SentenceTransformerEmbedder(server, model or SENTENCE_TRANSFORMERS_DEFAULT)
25
+ raise SemanticUnavailableError(
26
+ f"Unknown {server.var('EMBEDDING_BACKEND')} {backend!r}; expected 'fastembed' or "
27
+ "'sentence-transformers'."
28
+ )
29
+
30
+
31
+ class _FastEmbedEmbedder:
32
+ def __init__(self, server: Server, model_name: str) -> None:
33
+ try:
34
+ from fastembed import TextEmbedding
35
+ except ImportError as exc: # pragma: no cover - env-dependent
36
+ raise SemanticUnavailableError(install_hint(server, "fastembed")) from exc
37
+ self.model_name = model_name
38
+ self._model = TextEmbedding(model_name=model_name)
39
+ probe = next(iter(self._model.embed(["probe"])))
40
+ self.dim = len(probe)
41
+
42
+ def embed_documents(self, texts: list[str]) -> list[list[float]]:
43
+ return [[float(x) for x in vec] for vec in self._model.embed(texts)]
44
+
45
+ def embed_query(self, text: str) -> list[float]:
46
+ vec = next(iter(self._model.query_embed([text])))
47
+ return [float(x) for x in vec]
48
+
49
+
50
+ class _SentenceTransformerEmbedder:
51
+ def __init__(self, server: Server, model_name: str) -> None:
52
+ try:
53
+ from sentence_transformers import SentenceTransformer
54
+ except ImportError as exc: # pragma: no cover - env-dependent
55
+ raise SemanticUnavailableError(
56
+ install_hint(server, "sentence-transformers", extra="semantic-st")
57
+ ) from exc
58
+ self.model_name = model_name
59
+ self._model = SentenceTransformer(model_name)
60
+ self.dim = int(self._model.get_sentence_embedding_dimension())
61
+
62
+ def embed_documents(self, texts: list[str]) -> list[list[float]]:
63
+ vecs = self._model.encode(texts, normalize_embeddings=True)
64
+ return [[float(x) for x in vec] for vec in vecs]
65
+
66
+ def embed_query(self, text: str) -> list[float]:
67
+ vec = self._model.encode([text], normalize_embeddings=True)[0]
68
+ return [float(x) for x in vec]
@@ -0,0 +1,218 @@
1
+ """Keeps a server's index in step with its source, and searches it.
2
+
3
+ Pure orchestration: it takes a :class:`~.types.Source`, an :class:`~.types.Embedder`
4
+ and a :class:`~.types.VectorIndex` by injection and never imports an optional
5
+ dependency. Embedder and store calls run on worker threads, a lock keeps two syncs
6
+ from writing the store at once, and a semaphore caps the searches that embed and
7
+ query together.
8
+
9
+ The source decides what the index may hold: an item it hides is never embedded,
10
+ and one that becomes hidden is dropped by the next sync, before anything new is
11
+ embedded.
12
+
13
+ What a search returns does not rest on the index being clean, though. The index is
14
+ a file on disk that other processes can write, each with settings of its own (a
15
+ reindex run without the server's env file may have no filter and embed
16
+ everything). So with a filter active, every search asks the source which items are
17
+ visible now and returns only those, whatever the index holds.
18
+ """
19
+
20
+ import asyncio
21
+ from enum import Enum
22
+
23
+ from .chunking import chunk_text
24
+ from .sync import diff_versions
25
+ from .types import ChunkRecord, Embedder, ItemText, QueryHit, Skip, Source, SyncStats, VectorIndex
26
+
27
+ # Over-fetch factor, so keeping one hit per item still yields up to `limit` items.
28
+ QUERY_OVERFETCH = 4
29
+ # Searches that may embed and query at the same time.
30
+ SEARCH_CONCURRENCY = 2
31
+
32
+
33
+ class _Embedded(Enum):
34
+ """What became of one item a sync looked at."""
35
+
36
+ TEXT = "text" # its chunks are in the index
37
+ EMPTY = "empty" # it has no text: marked, so the next sync leaves it alone
38
+ HIDDEN = "hidden" # the source hides it now: nothing kept
39
+ NOT_READY = "not-ready" # not readable yet: not marked, so the next sync looks again
40
+
41
+
42
+ class Engine:
43
+ """Keeps the index in step with the source and answers searches."""
44
+
45
+ def __init__(
46
+ self,
47
+ source: Source,
48
+ embedder: Embedder,
49
+ index: VectorIndex,
50
+ *,
51
+ auto_sync: bool = True,
52
+ ) -> None:
53
+ self._source = source
54
+ self._embedder = embedder
55
+ self._index = index
56
+ self._auto_sync = auto_sync
57
+ self._lock = asyncio.Lock()
58
+ # Query embedding runs on worker threads; a remote caller firing many
59
+ # searches at once could otherwise tie up the model and the CPU.
60
+ self._search_slots = asyncio.Semaphore(SEARCH_CONCURRENCY)
61
+
62
+ async def sync(self, *, full: bool = False) -> SyncStats:
63
+ """Re-embed changed items (or rebuild everything)."""
64
+ async with self._lock:
65
+ stats, _ = await self._sync_locked(full=full)
66
+ return stats
67
+
68
+ async def prune(self) -> int:
69
+ """Drop indexed items that are hidden or gone, embedding nothing.
70
+
71
+ The cheap half of a sync (one listing, no model). Returns how many items
72
+ with text left the index.
73
+ """
74
+ current = await self._source.list_versions()
75
+ async with self._lock:
76
+ return await self._drop_missing(current)
77
+
78
+ async def _drop_missing(self, current: dict[str, str]) -> int:
79
+ """Remove every item that isn't in ``current``. Call with the lock."""
80
+ indexed = await asyncio.to_thread(self._index.item_versions)
81
+ empty = await asyncio.to_thread(self._index.empty_versions)
82
+ gone = [item_id for item_id in {**empty, **indexed} if item_id not in current]
83
+ if gone:
84
+ await asyncio.to_thread(self._index.delete_items, gone)
85
+ return sum(1 for item_id in gone if item_id in indexed)
86
+
87
+ def _needs_rebuild(self) -> bool:
88
+ """The stored vectors come from another model, or have another width."""
89
+ if self._index.model_name != self._embedder.model_name:
90
+ return True
91
+ return bool(self._index.dim) and self._index.dim != self._embedder.dim
92
+
93
+ async def _sync_locked(self, *, full: bool) -> tuple[SyncStats, set[str]]:
94
+ """Sync, and say which items were visible when it looked. Call with the lock."""
95
+ # Ask the source first: if it is down, or its filter doesn't resolve, the
96
+ # index is left exactly as it was (a full rebuild included).
97
+ current = await self._source.list_versions()
98
+
99
+ model = self._embedder.model_name
100
+ if full or self._needs_rebuild():
101
+ await asyncio.to_thread(self._index.reset, model)
102
+ indexed: dict[str, str] = {}
103
+ empty: dict[str, str] = {}
104
+ else:
105
+ indexed = await asyncio.to_thread(self._index.item_versions)
106
+ empty = await asyncio.to_thread(self._index.empty_versions)
107
+
108
+ # An item with no text was looked at once for its version and marked, so it
109
+ # isn't read again until it changes.
110
+ plan = diff_versions(current, {**empty, **indexed})
111
+
112
+ # Drop before embedding: an item that is now hidden must leave the index even
113
+ # if an embedding below fails.
114
+ if plan.deleted_ids:
115
+ await asyncio.to_thread(self._index.delete_items, plan.deleted_ids)
116
+
117
+ added = 0
118
+ updated = 0
119
+ for item_id in plan.changed_ids:
120
+ outcome = await self._embed_item(item_id, current[item_id])
121
+ if item_id in indexed:
122
+ # Its text changed, or it was emptied: either way its entry did.
123
+ if outcome in (_Embedded.TEXT, _Embedded.EMPTY):
124
+ updated += 1
125
+ elif outcome is _Embedded.TEXT: # new, or an empty item gained text
126
+ added += 1
127
+
128
+ stats = SyncStats(
129
+ added=added,
130
+ updated=updated,
131
+ deleted=sum(1 for item_id in plan.deleted_ids if item_id in indexed),
132
+ total=len(current),
133
+ model_name=model,
134
+ )
135
+ return stats, set(current)
136
+
137
+ async def _embed_item(self, item_id: str, version: str) -> _Embedded:
138
+ """(Re)embed one item, or mark it as having no text, or leave it out."""
139
+ item = await self._source.get_item(item_id)
140
+
141
+ # Drop any stale chunks (and mark) first, so an item that shrank leaves
142
+ # nothing behind.
143
+ await asyncio.to_thread(self._index.delete_items, [item_id])
144
+ if item is Skip.HIDDEN:
145
+ return _Embedded.HIDDEN
146
+ if item is Skip.NOT_READY:
147
+ return _Embedded.NOT_READY
148
+ assert isinstance(item, ItemText)
149
+
150
+ chunks = chunk_text(item.title, item.text)
151
+ if not chunks:
152
+ await asyncio.to_thread(self._index.mark_empty, item_id, version)
153
+ return _Embedded.EMPTY
154
+
155
+ embeddings = await asyncio.to_thread(self._embedder.embed_documents, chunks)
156
+ records = [
157
+ ChunkRecord(
158
+ chunk_id=f"{item_id}::{i}",
159
+ item_id=item_id,
160
+ title=item.title,
161
+ version=version,
162
+ chunk_index=i,
163
+ text=text,
164
+ embedding=embedding,
165
+ )
166
+ for i, (text, embedding) in enumerate(zip(chunks, embeddings, strict=True))
167
+ ]
168
+ await asyncio.to_thread(self._index.upsert, records)
169
+ return _Embedded.TEXT
170
+
171
+ async def _visible_for_search(self) -> set[str] | None:
172
+ """Bring the index up to date for a search; the visible ids, or None for all.
173
+
174
+ With auto-sync, a sync. Without it, nothing, unless a filter is active: the
175
+ filter still has to hold now, so the source is asked which items are
176
+ visible, and the index is tidied if no sync is using it (a running sync
177
+ tidies it anyway, and a search shouldn't wait behind a long one).
178
+ """
179
+ if self._auto_sync:
180
+ async with self._lock:
181
+ _, visible = await self._sync_locked(full=False)
182
+ return visible if self._source.filter_active else None
183
+ if not self._source.filter_active:
184
+ return None
185
+ current = await self._source.list_versions()
186
+ if not self._lock.locked():
187
+ async with self._lock:
188
+ await self._drop_missing(current)
189
+ return set(current)
190
+
191
+ @staticmethod
192
+ def _best(hits: list[QueryHit], visible: set[str] | None, limit: int) -> list[QueryHit]:
193
+ # Trust the source's answer, not the index's contents: another process may
194
+ # have written hidden items into the same index since it was tidied.
195
+ if visible is not None:
196
+ hits = [hit for hit in hits if hit.item_id in visible]
197
+ best: dict[str, QueryHit] = {}
198
+ for hit in hits: # hits are ordered best-first
199
+ best.setdefault(hit.item_id, hit)
200
+ return list(best.values())[:limit]
201
+
202
+ async def search(self, query: str, limit: int = 10) -> list[QueryHit]:
203
+ """Up to ``limit`` items closest in meaning to ``query``: each one's best chunk."""
204
+ visible = await self._visible_for_search()
205
+ async with self._search_slots:
206
+ embedding = await asyncio.to_thread(self._embedder.embed_query, query)
207
+ hits = await asyncio.to_thread(self._index.query, embedding, limit * QUERY_OVERFETCH)
208
+ return self._best(hits, visible, limit)
209
+
210
+ async def word_search(self, query: str, limit: int = 10) -> list[QueryHit]:
211
+ """Up to ``limit`` items with every word of ``query``, ignoring case and accents.
212
+
213
+ Raises :class:`~second_brain_core.errors.WordSearchUnavailableError` when the
214
+ store has no word index.
215
+ """
216
+ visible = await self._visible_for_search()
217
+ hits = await asyncio.to_thread(self._index.word_search, query, limit * QUERY_OVERFETCH)
218
+ return self._best(hits, visible, limit)