atomzero 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atomzero/__init__.py +8 -0
- atomzero/agent_runtime.py +225 -0
- atomzero/chat_human.py +486 -0
- atomzero/chat_server.py +2494 -0
- atomzero/curiosity.py +66 -0
- atomzero/doc_ingest.py +312 -0
- atomzero/doc_memory.py +1112 -0
- atomzero/feedback.py +188 -0
- atomzero/knowledge_gaps.py +157 -0
- atomzero/lora.py +140 -0
- atomzero/ml_safety.py +165 -0
- atomzero/neural_embedder.py +108 -0
- atomzero/research_queue.py +330 -0
- atomzero/safety.py +226 -0
- atomzero/teacher_notes.py +78 -0
- atomzero/teacher_provider.py +104 -0
- atomzero/tool_planner.py +207 -0
- atomzero/transformer.py +102 -0
- atomzero/web_research.py +165 -0
- atomzero-0.1.0.dist-info/METADATA +96 -0
- atomzero-0.1.0.dist-info/RECORD +26 -0
- atomzero-0.1.0.dist-info/WHEEL +5 -0
- atomzero-0.1.0.dist-info/entry_points.txt +2 -0
- atomzero-0.1.0.dist-info/licenses/LICENSE +202 -0
- atomzero-0.1.0.dist-info/licenses/NOTICE +24 -0
- atomzero-0.1.0.dist-info/top_level.txt +1 -0
atomzero/doc_memory.py
ADDED
|
@@ -0,0 +1,1112 @@
|
|
|
1
|
+
"""Local document memory with SQLite FTS5 and provenance.
|
|
2
|
+
|
|
3
|
+
Documents are chunked, indexed and retrieved with their source. Facts stay in
|
|
4
|
+
this index, not in the model weights.
|
|
5
|
+
|
|
6
|
+
python -m atomzero.doc_memory add notes.txt --db data/docs.db
|
|
7
|
+
python -m atomzero.doc_memory --db data/docs.db search "kira"
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import array
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
import sqlite3
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
import zlib
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
import numpy as np
|
|
25
|
+
except ImportError: # pragma: no cover - numpy is a declared dependency
|
|
26
|
+
np = None
|
|
27
|
+
|
|
28
|
+
MAX_CHARS = 900
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SourceNotFound(ValueError):
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
STOPWORDS = frozenset({
|
|
36
|
+
"a", "an", "and", "are", "as", "at", "be", "been", "being", "but",
|
|
37
|
+
"by", "can", "could", "did", "do", "does", "for", "from", "had", "has",
|
|
38
|
+
"have", "he", "her", "here", "him", "his", "how", "i", "if", "in",
|
|
39
|
+
"into", "is", "it", "its", "me", "my", "no", "not", "of", "on", "or",
|
|
40
|
+
"our", "please", "she", "should", "so", "tell", "than", "that", "the",
|
|
41
|
+
"their", "them", "then", "there", "these", "they", "this", "those",
|
|
42
|
+
"to", "was", "we", "were", "what", "when", "where", "which", "who",
|
|
43
|
+
"why", "will", "with", "would", "you", "your",
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _split_long(paragraph: str) -> list[str]:
|
|
48
|
+
"""Split a long paragraph near a word boundary, never mid-word if possible."""
|
|
49
|
+
out = []
|
|
50
|
+
while len(paragraph) > MAX_CHARS:
|
|
51
|
+
cut = paragraph.rfind(" ", 0, MAX_CHARS)
|
|
52
|
+
if cut < MAX_CHARS // 2:
|
|
53
|
+
cut = MAX_CHARS
|
|
54
|
+
piece = paragraph[:cut].strip()
|
|
55
|
+
if piece:
|
|
56
|
+
out.append(piece)
|
|
57
|
+
paragraph = paragraph[cut:].lstrip()
|
|
58
|
+
if paragraph.strip():
|
|
59
|
+
out.append(paragraph.strip())
|
|
60
|
+
return out
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
EMBED_DIM = 512
|
|
64
|
+
HASHED_CACHE_KEY = "hashed-ngram-v1"
|
|
65
|
+
# Minimum cosine for a semantic-only candidate. Lexical FTS hits are always
|
|
66
|
+
# kept; this only stops unrelated chunks from being dragged into the pool.
|
|
67
|
+
HYBRID_MIN_HASHED = 0.28
|
|
68
|
+
HYBRID_MIN_NEURAL = 0.30
|
|
69
|
+
ANN_MIN_CHUNKS = 256
|
|
70
|
+
ANN_BUILD_ITERATIONS = 8
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class _AnnFallback(RuntimeError):
|
|
74
|
+
"""Internal signal: use the exact matrix for this query."""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _char_ngrams(text: str) -> list[str]:
|
|
78
|
+
words = re.findall(r"[^\W_]+", str(text).lower(), re.UNICODE)
|
|
79
|
+
grams = []
|
|
80
|
+
for word in words:
|
|
81
|
+
padded = f" {word} "
|
|
82
|
+
for size in (3, 4, 5):
|
|
83
|
+
grams.extend(padded[index:index + size]
|
|
84
|
+
for index in range(len(padded) - size + 1))
|
|
85
|
+
return grams
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def embed_text(text: str, dim: int = EMBED_DIM) -> list[float]:
|
|
89
|
+
"""Deterministic hashed character n-gram embedding (no neural model)."""
|
|
90
|
+
vector = [0.0] * dim
|
|
91
|
+
for gram in _char_ngrams(text):
|
|
92
|
+
vector[zlib.crc32(gram.encode("utf-8")) % dim] += 1.0
|
|
93
|
+
norm = math.sqrt(sum(value * value for value in vector))
|
|
94
|
+
if norm:
|
|
95
|
+
vector = [value / norm for value in vector]
|
|
96
|
+
return vector
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def embedding_similarity(left: str, right: str) -> float:
|
|
100
|
+
a, b = embed_text(left), embed_text(right)
|
|
101
|
+
return sum(x * y for x, y in zip(a, b))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _query_terms(text: str) -> set[str]:
|
|
105
|
+
words = re.findall(r"[^\W_]+", str(text).lower(), re.UNICODE)
|
|
106
|
+
return {word for word in words
|
|
107
|
+
if len(word) > 2 and word not in STOPWORDS}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _pack_vector(values) -> bytes:
|
|
111
|
+
return array.array("f", (float(value) for value in values)).tobytes()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _unpack_vector(blob: bytes):
|
|
115
|
+
values = array.array("f")
|
|
116
|
+
values.frombytes(blob)
|
|
117
|
+
return values
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _normalized_rows(matrix):
|
|
121
|
+
norms = np.linalg.norm(matrix, axis=1, keepdims=True)
|
|
122
|
+
norms[norms == 0.0] = 1.0
|
|
123
|
+
return matrix / norms
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def build_ivf(matrix, nlist: int | None = None,
|
|
127
|
+
iterations: int = ANN_BUILD_ITERATIONS) -> dict | None:
|
|
128
|
+
"""Deterministic k-means IVF index over normalized rows.
|
|
129
|
+
|
|
130
|
+
This is approximate retrieval: a query scores cluster centroids, probes
|
|
131
|
+
the best `nprobe` buckets, then reranks only their members exactly.
|
|
132
|
+
"""
|
|
133
|
+
if np is None or matrix is None or len(matrix) < 2:
|
|
134
|
+
return None
|
|
135
|
+
n = len(matrix)
|
|
136
|
+
if nlist is None:
|
|
137
|
+
nlist = min(64, max(2, int(round(math.sqrt(n)))))
|
|
138
|
+
nlist = min(max(2, int(nlist)), n)
|
|
139
|
+
starts = np.linspace(0, n - 1, nlist).astype(int)
|
|
140
|
+
centroids = matrix[starts].copy()
|
|
141
|
+
for _ in range(max(1, iterations)):
|
|
142
|
+
assignments = (matrix @ centroids.T).argmax(axis=1)
|
|
143
|
+
updated = centroids.copy()
|
|
144
|
+
for cluster in range(nlist):
|
|
145
|
+
members = matrix[assignments == cluster]
|
|
146
|
+
if len(members):
|
|
147
|
+
updated[cluster] = members.mean(axis=0)
|
|
148
|
+
centroids = _normalized_rows(updated)
|
|
149
|
+
assignments = (matrix @ centroids.T).argmax(axis=1)
|
|
150
|
+
buckets = [np.where(assignments == cluster)[0].astype(np.int32)
|
|
151
|
+
for cluster in range(nlist)]
|
|
152
|
+
return {"centroids": centroids, "buckets": buckets, "nlist": nlist}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def ann_topk(matrix, ann: dict, query_vector, limit: int,
|
|
156
|
+
probe: int | None = None):
|
|
157
|
+
"""Return exact-scored top-k within the best `probe` IVF buckets."""
|
|
158
|
+
if ann is None or matrix is None:
|
|
159
|
+
raise _AnnFallback("no ANN index")
|
|
160
|
+
nlist = int(ann["nlist"])
|
|
161
|
+
if probe is None:
|
|
162
|
+
probe = max(1, int(math.ceil(math.sqrt(nlist))))
|
|
163
|
+
probe = min(nlist, max(1, int(probe)))
|
|
164
|
+
cluster_scores = ann["centroids"] @ query_vector
|
|
165
|
+
clusters = np.argsort(-cluster_scores, kind="stable")[:probe]
|
|
166
|
+
parts = [ann["buckets"][int(cluster)] for cluster in clusters
|
|
167
|
+
if len(ann["buckets"][int(cluster)])]
|
|
168
|
+
if not parts:
|
|
169
|
+
raise _AnnFallback("empty ANN buckets")
|
|
170
|
+
positions = np.concatenate(parts)
|
|
171
|
+
if len(positions) < min(limit, len(matrix)):
|
|
172
|
+
raise _AnnFallback("too few ANN candidates")
|
|
173
|
+
scores = matrix[positions] @ query_vector
|
|
174
|
+
order = np.argsort(-scores, kind="stable")[:limit]
|
|
175
|
+
return positions[order], scores[order]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def rerank_hits(query: str, hits: list[dict], k: int = 5) -> list[dict]:
|
|
179
|
+
"""Lexically rerank FTS candidates by coverage, title, phrase and bm25.
|
|
180
|
+
|
|
181
|
+
This is a deterministic reranker, not an embedding model. It is intentionally
|
|
182
|
+
cheap: local search should stay fast and work without a second model.
|
|
183
|
+
"""
|
|
184
|
+
terms = _query_terms(query)
|
|
185
|
+
if not terms or not hits:
|
|
186
|
+
return hits[:k]
|
|
187
|
+
phrase = " ".join(str(query or "").lower().split())
|
|
188
|
+
query_vector = embed_text(query)
|
|
189
|
+
raw = []
|
|
190
|
+
for hit in hits:
|
|
191
|
+
value = hit.get("score")
|
|
192
|
+
try:
|
|
193
|
+
number = float(value) if value is not None else None
|
|
194
|
+
except (TypeError, ValueError):
|
|
195
|
+
number = None
|
|
196
|
+
raw.append(number if number is not None and math.isfinite(number)
|
|
197
|
+
else None)
|
|
198
|
+
finite = [value for value in raw if value is not None]
|
|
199
|
+
# bm25 returns lower-is-better; normalize it into 0..1. Embedding-only
|
|
200
|
+
# candidates have no bm25 and get 0 for this component.
|
|
201
|
+
high, low = (max(finite), min(finite)) if finite else (0.0, 0.0)
|
|
202
|
+
span = high - low
|
|
203
|
+
scored = []
|
|
204
|
+
for index, hit in enumerate(hits):
|
|
205
|
+
chunk = str(hit.get("chunk", "")).lower()
|
|
206
|
+
title = str(hit.get("title", "")).lower()
|
|
207
|
+
section = str(hit.get("section", "")).strip().lower()
|
|
208
|
+
searchable = f"{title} {section} {chunk}"
|
|
209
|
+
words = _query_terms(searchable)
|
|
210
|
+
coverage = len(terms & words) / len(terms)
|
|
211
|
+
title_hits = len(terms & _query_terms(f"{title} {section}")) / len(terms)
|
|
212
|
+
section_hits = (len(terms & _query_terms(section)) / len(terms)
|
|
213
|
+
if section else 0.0)
|
|
214
|
+
exact = 1.0 if len(terms) >= 2 and phrase in searchable else 0.0
|
|
215
|
+
if raw[index] is not None and span:
|
|
216
|
+
bm25 = 1.0 - ((raw[index] - low) / span)
|
|
217
|
+
elif raw[index] is not None:
|
|
218
|
+
bm25 = 0.5
|
|
219
|
+
else:
|
|
220
|
+
bm25 = 0.0
|
|
221
|
+
embedding = hit.get("embedding_score")
|
|
222
|
+
if embedding is None:
|
|
223
|
+
body_vector = embed_text(searchable)
|
|
224
|
+
embedding = sum(x * y for x, y in zip(query_vector, body_vector))
|
|
225
|
+
score = (0.40 * coverage + 0.10 * title_hits
|
|
226
|
+
+ 0.05 * section_hits + 0.10 * exact
|
|
227
|
+
+ 0.15 * bm25 + 0.25 * embedding)
|
|
228
|
+
item = dict(hit)
|
|
229
|
+
item["rerank_score"] = score
|
|
230
|
+
item["embedding_score"] = embedding
|
|
231
|
+
scored.append((score, index, item))
|
|
232
|
+
scored.sort(key=lambda row: (-row[0], row[1]))
|
|
233
|
+
return [item for _, _, item in scored[:k]]
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def best_snippet(query: str, chunk: str, max_chars: int = 500) -> str:
|
|
237
|
+
"""Select the one or two sentences that best match the query."""
|
|
238
|
+
text = str(chunk or "").strip()
|
|
239
|
+
if not text:
|
|
240
|
+
return ""
|
|
241
|
+
parts = [part.strip() for part in
|
|
242
|
+
re.split(r"(?<=[.!?])\s+|\n+", text) if part.strip()]
|
|
243
|
+
if len(parts) <= 1:
|
|
244
|
+
return text[:max_chars]
|
|
245
|
+
words = {word for word in re.findall(r"[a-z0-9]+", str(query).lower())
|
|
246
|
+
if len(word) > 2 and word not in STOPWORDS}
|
|
247
|
+
scores = [
|
|
248
|
+
sum(1 for word in re.findall(r"[a-z0-9]+", part.lower())
|
|
249
|
+
if word in words)
|
|
250
|
+
for part in parts
|
|
251
|
+
]
|
|
252
|
+
chosen = sorted(sorted(range(len(parts)),
|
|
253
|
+
key=lambda i: (-scores[i], i))[:2])
|
|
254
|
+
snippet = " ".join(parts[i] for i in chosen)
|
|
255
|
+
if len(snippet) > max_chars:
|
|
256
|
+
snippet = parts[min(range(len(parts)),
|
|
257
|
+
key=lambda i: (-scores[i], i))]
|
|
258
|
+
return snippet[:max_chars]
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def format_citation(hit: dict) -> str:
|
|
262
|
+
"""Format a hit as ``source (p. N) [section]`` for chat/UI display."""
|
|
263
|
+
location = str(hit.get("source_url") or hit.get("source") or "")
|
|
264
|
+
page = hit.get("page")
|
|
265
|
+
if page not in (None, ""):
|
|
266
|
+
location += f" (p. {page})"
|
|
267
|
+
section = str(hit.get("section") or "").strip()
|
|
268
|
+
if section:
|
|
269
|
+
location += f" [{section}]"
|
|
270
|
+
return location
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def add_citation_source(sources: dict, hit: dict) -> dict:
|
|
274
|
+
"""Merge a page/section hit into a source-keyed citation map."""
|
|
275
|
+
key = str(hit.get("source", ""))
|
|
276
|
+
source = sources.get(key)
|
|
277
|
+
if source is None:
|
|
278
|
+
source = {"source": hit.get("source", ""),
|
|
279
|
+
"title": hit.get("title", "")}
|
|
280
|
+
for field in ("kind", "source_url", "teacher", "model", "verified",
|
|
281
|
+
"reviewer", "verified_at"):
|
|
282
|
+
if field in hit:
|
|
283
|
+
source[field] = hit[field]
|
|
284
|
+
sources[key] = source
|
|
285
|
+
page = hit.get("page")
|
|
286
|
+
if page not in (None, ""):
|
|
287
|
+
pages = source.setdefault("pages", [])
|
|
288
|
+
if page not in pages:
|
|
289
|
+
pages.append(page)
|
|
290
|
+
section = str(hit.get("section") or "").strip()
|
|
291
|
+
if section:
|
|
292
|
+
sections = source.setdefault("sections", [])
|
|
293
|
+
if section not in sections:
|
|
294
|
+
sections.append(section)
|
|
295
|
+
return source
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _chunks(text: str) -> list[str]:
|
|
299
|
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
300
|
+
paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
|
301
|
+
out = []
|
|
302
|
+
current = ""
|
|
303
|
+
for para in paragraphs:
|
|
304
|
+
if len(para) > MAX_CHARS:
|
|
305
|
+
if current:
|
|
306
|
+
out.append(current)
|
|
307
|
+
current = ""
|
|
308
|
+
out.extend(_split_long(para))
|
|
309
|
+
continue
|
|
310
|
+
if len(current) + len(para) + 2 <= MAX_CHARS:
|
|
311
|
+
current = (current + "\n\n" + para).strip()
|
|
312
|
+
else:
|
|
313
|
+
if current:
|
|
314
|
+
out.append(current)
|
|
315
|
+
current = para
|
|
316
|
+
if current:
|
|
317
|
+
out.append(current)
|
|
318
|
+
return out
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
class DocMemory:
|
|
322
|
+
def __init__(self, path: str | Path = "data/docs.db"):
|
|
323
|
+
self.path = Path(path)
|
|
324
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
325
|
+
self._lock = threading.RLock()
|
|
326
|
+
self.reranker = None
|
|
327
|
+
self._vector_index = None
|
|
328
|
+
self.vector_index_mode = "auto" # auto | ann | exact
|
|
329
|
+
self.ann_min_chunks = ANN_MIN_CHUNKS
|
|
330
|
+
self.ann_probe = None
|
|
331
|
+
self._vector_fallback_count = 0
|
|
332
|
+
self.con = sqlite3.connect(str(self.path), check_same_thread=False)
|
|
333
|
+
self.con.execute(
|
|
334
|
+
"CREATE VIRTUAL TABLE IF NOT EXISTS docs USING fts5("
|
|
335
|
+
"chunk, source UNINDEXED, title UNINDEXED, sha UNINDEXED, "
|
|
336
|
+
"ts UNINDEXED)")
|
|
337
|
+
self.con.execute(
|
|
338
|
+
"CREATE TABLE IF NOT EXISTS doc_meta("
|
|
339
|
+
"source TEXT PRIMARY KEY, kind TEXT NOT NULL, "
|
|
340
|
+
"source_url TEXT NOT NULL, teacher TEXT NOT NULL, "
|
|
341
|
+
"model TEXT NOT NULL, verified INTEGER NOT NULL, "
|
|
342
|
+
"created TEXT NOT NULL, reviewer TEXT NOT NULL DEFAULT '', "
|
|
343
|
+
"verified_at TEXT NOT NULL DEFAULT '')")
|
|
344
|
+
self.con.execute(
|
|
345
|
+
"CREATE TABLE IF NOT EXISTS doc_vectors("
|
|
346
|
+
"doc_rowid INTEGER NOT NULL, model TEXT NOT NULL, "
|
|
347
|
+
"dim INTEGER NOT NULL, vec BLOB NOT NULL, "
|
|
348
|
+
"PRIMARY KEY(doc_rowid, model))")
|
|
349
|
+
self.con.execute(
|
|
350
|
+
"CREATE TABLE IF NOT EXISTS doc_chunk_meta("
|
|
351
|
+
"doc_rowid INTEGER PRIMARY KEY, page INTEGER, "
|
|
352
|
+
"section TEXT NOT NULL DEFAULT '', "
|
|
353
|
+
"char_start INTEGER, char_end INTEGER)")
|
|
354
|
+
columns = {row[1] for row in self.con.execute(
|
|
355
|
+
"PRAGMA table_info(doc_meta)")}
|
|
356
|
+
for name, ddl in (
|
|
357
|
+
("reviewer", "TEXT NOT NULL DEFAULT ''"),
|
|
358
|
+
("verified_at", "TEXT NOT NULL DEFAULT ''")):
|
|
359
|
+
if name not in columns:
|
|
360
|
+
self.con.execute(
|
|
361
|
+
f"ALTER TABLE doc_meta ADD COLUMN {name} {ddl}")
|
|
362
|
+
self.con.commit()
|
|
363
|
+
|
|
364
|
+
def add_text(self, source: str, text: str, title: str = "",
|
|
365
|
+
replace: bool = True, meta: dict | None = None) -> int:
|
|
366
|
+
return self.add_blocks(
|
|
367
|
+
source, [{"text": text, "page": None, "section": ""}],
|
|
368
|
+
title=title, replace=replace, meta=meta)
|
|
369
|
+
|
|
370
|
+
def add_blocks(self, source: str, blocks: list[dict], title: str = "",
|
|
371
|
+
replace: bool = True, meta: dict | None = None) -> int:
|
|
372
|
+
"""Index text blocks while keeping page/section provenance.
|
|
373
|
+
|
|
374
|
+
A block is a piece of text with an optional 1-based ``page`` and a
|
|
375
|
+
``section`` label. Each block is split with the normal chunker; the
|
|
376
|
+
resulting chunks inherit the block's provenance and a character range
|
|
377
|
+
inside that block.
|
|
378
|
+
"""
|
|
379
|
+
source = str(source)
|
|
380
|
+
title = str(title or Path(source).stem)
|
|
381
|
+
meta = meta or {}
|
|
382
|
+
kind = str(meta.get("kind", "document"))
|
|
383
|
+
source_url = str(meta.get("source_url", ""))
|
|
384
|
+
teacher = str(meta.get("teacher", ""))
|
|
385
|
+
model = str(meta.get("model", ""))
|
|
386
|
+
verified = 1 if meta.get("verified", True) else 0
|
|
387
|
+
created = str(meta.get("created", int(time.time())))
|
|
388
|
+
reviewer = str(meta.get("reviewer", ""))
|
|
389
|
+
verified_at = str(meta.get("verified_at", ""))
|
|
390
|
+
block_texts = [str(block.get("text", "")) for block in blocks]
|
|
391
|
+
sha = hashlib.sha256(
|
|
392
|
+
"\n\n".join(block_texts).encode("utf-8")).hexdigest()
|
|
393
|
+
with self._lock:
|
|
394
|
+
if replace:
|
|
395
|
+
self.con.execute("DELETE FROM docs WHERE source = ?",
|
|
396
|
+
(source,))
|
|
397
|
+
added = 0
|
|
398
|
+
for block in blocks:
|
|
399
|
+
text = str(block.get("text", "")).replace(
|
|
400
|
+
"\r\n", "\n").replace("\r", "\n").strip()
|
|
401
|
+
if not text:
|
|
402
|
+
continue
|
|
403
|
+
page = block.get("page")
|
|
404
|
+
try:
|
|
405
|
+
page = int(page) if page not in (None, "") else None
|
|
406
|
+
except (TypeError, ValueError):
|
|
407
|
+
page = None
|
|
408
|
+
section = str(block.get("section", "") or "").strip()
|
|
409
|
+
cursor = 0
|
|
410
|
+
for chunk in _chunks(text):
|
|
411
|
+
position = text.find(chunk, cursor)
|
|
412
|
+
if position < 0:
|
|
413
|
+
position = cursor
|
|
414
|
+
char_start = position
|
|
415
|
+
char_end = position + len(chunk)
|
|
416
|
+
inserted = self.con.execute(
|
|
417
|
+
"INSERT INTO docs(chunk, source, title, sha, ts) "
|
|
418
|
+
"VALUES (?, ?, ?, ?, ?)",
|
|
419
|
+
(chunk, source, title, sha, int(time.time())))
|
|
420
|
+
rowid = inserted.lastrowid
|
|
421
|
+
if not rowid:
|
|
422
|
+
rowid = self.con.execute(
|
|
423
|
+
"SELECT last_insert_rowid()").fetchone()[0]
|
|
424
|
+
self.con.execute(
|
|
425
|
+
"INSERT OR REPLACE INTO doc_chunk_meta("
|
|
426
|
+
"doc_rowid, page, section, char_start, char_end) "
|
|
427
|
+
"VALUES (?, ?, ?, ?, ?)",
|
|
428
|
+
(rowid, page, section, char_start, char_end))
|
|
429
|
+
cursor = char_end
|
|
430
|
+
added += 1
|
|
431
|
+
self.con.execute(
|
|
432
|
+
"INSERT OR REPLACE INTO doc_meta("
|
|
433
|
+
"source, kind, source_url, teacher, model, verified, created, "
|
|
434
|
+
"reviewer, verified_at) "
|
|
435
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
436
|
+
(source, kind, source_url, teacher, model, verified, created,
|
|
437
|
+
reviewer, verified_at))
|
|
438
|
+
if replace:
|
|
439
|
+
self._drop_stale_vectors()
|
|
440
|
+
self._drop_stale_chunk_meta()
|
|
441
|
+
self._vector_index = None
|
|
442
|
+
self.con.commit()
|
|
443
|
+
return added
|
|
444
|
+
|
|
445
|
+
def add_file(self, path: str | Path, title: str = "",
|
|
446
|
+
meta: dict | None = None) -> int:
|
|
447
|
+
from atomzero.doc_ingest import extract_file
|
|
448
|
+
|
|
449
|
+
p = Path(path)
|
|
450
|
+
extracted = extract_file(p, title=title)
|
|
451
|
+
return self.add_blocks(
|
|
452
|
+
str(p), extracted["blocks"], title=extracted["title"],
|
|
453
|
+
meta=meta)
|
|
454
|
+
|
|
455
|
+
def _drop_stale_vectors(self) -> None:
|
|
456
|
+
self.con.execute(
|
|
457
|
+
"DELETE FROM doc_vectors WHERE doc_rowid NOT IN "
|
|
458
|
+
"(SELECT rowid FROM docs)")
|
|
459
|
+
|
|
460
|
+
def _drop_stale_chunk_meta(self) -> None:
|
|
461
|
+
self.con.execute(
|
|
462
|
+
"DELETE FROM doc_chunk_meta WHERE doc_rowid NOT IN "
|
|
463
|
+
"(SELECT rowid FROM docs)")
|
|
464
|
+
|
|
465
|
+
@staticmethod
|
|
466
|
+
def _vector_model_key(active) -> str:
|
|
467
|
+
if active is None or not hasattr(active, "embed"):
|
|
468
|
+
return HASHED_CACHE_KEY
|
|
469
|
+
key = getattr(active, "cache_key", "")
|
|
470
|
+
if key:
|
|
471
|
+
return str(key)
|
|
472
|
+
path = getattr(active, "model_path", "")
|
|
473
|
+
if path:
|
|
474
|
+
return f"neural:{Path(path).resolve()}"
|
|
475
|
+
return "neural:custom"
|
|
476
|
+
|
|
477
|
+
@staticmethod
|
|
478
|
+
def _embed_texts(active, texts: list[str]) -> list[list[float]]:
|
|
479
|
+
if active is not None and hasattr(active, "embed"):
|
|
480
|
+
vectors = active.embed(texts)
|
|
481
|
+
return [[float(value) for value in row] for row in vectors.tolist()]
|
|
482
|
+
return [embed_text(text) for text in texts]
|
|
483
|
+
|
|
484
|
+
def _vectors_for_rows(self, active, rows) -> dict[int, list[float]]:
|
|
485
|
+
"""Load cached chunk vectors, computing and storing missing ones."""
|
|
486
|
+
if not rows:
|
|
487
|
+
return {}
|
|
488
|
+
key = self._vector_model_key(active)
|
|
489
|
+
wanted = {rowid: text for rowid, text in rows}
|
|
490
|
+
vectors: dict[int, list[float]] = {}
|
|
491
|
+
ids = list(wanted)
|
|
492
|
+
for start in range(0, len(ids), 400):
|
|
493
|
+
batch = ids[start:start + 400]
|
|
494
|
+
placeholders = ",".join("?" for _ in batch)
|
|
495
|
+
found = self.con.execute(
|
|
496
|
+
"SELECT doc_rowid, vec FROM doc_vectors "
|
|
497
|
+
f"WHERE model = ? AND doc_rowid IN ({placeholders})",
|
|
498
|
+
(key, *batch)).fetchall()
|
|
499
|
+
for rowid, blob in found:
|
|
500
|
+
vectors[rowid] = _unpack_vector(blob)
|
|
501
|
+
missing = [rowid for rowid in ids if rowid not in vectors]
|
|
502
|
+
if missing:
|
|
503
|
+
computed = self._embed_texts(
|
|
504
|
+
active, [wanted[rowid] for rowid in missing])
|
|
505
|
+
self.con.executemany(
|
|
506
|
+
"INSERT OR REPLACE INTO doc_vectors"
|
|
507
|
+
"(doc_rowid, model, dim, vec) VALUES (?, ?, ?, ?)",
|
|
508
|
+
[(rowid, key, len(vector), _pack_vector(vector))
|
|
509
|
+
for rowid, vector in zip(missing, computed)])
|
|
510
|
+
self.con.commit()
|
|
511
|
+
for rowid, vector in zip(missing, computed):
|
|
512
|
+
vectors[rowid] = vector
|
|
513
|
+
return vectors
|
|
514
|
+
|
|
515
|
+
def prewarm_vectors(self, active=None) -> dict:
|
|
516
|
+
"""Build the in-memory matrix over all current chunk vectors.
|
|
517
|
+
|
|
518
|
+
With no explicit reranker, use the attached one (neural when
|
|
519
|
+
configured, hashed otherwise).
|
|
520
|
+
"""
|
|
521
|
+
if active is None:
|
|
522
|
+
active = self.reranker
|
|
523
|
+
started = time.perf_counter()
|
|
524
|
+
with self._lock:
|
|
525
|
+
rows = self.con.execute(
|
|
526
|
+
"SELECT rowid, chunk, source, title FROM docs "
|
|
527
|
+
"ORDER BY rowid").fetchall()
|
|
528
|
+
key = self._vector_model_key(active)
|
|
529
|
+
if not rows:
|
|
530
|
+
self._vector_index = {
|
|
531
|
+
"key": key, "rows": [], "rowids": [], "matrix": None,
|
|
532
|
+
"positions": {}, "build_s": 0.0}
|
|
533
|
+
return self.vector_index_stats()
|
|
534
|
+
texts = [(row[0], f"{row[3]} {row[1]}".strip()) for row in rows]
|
|
535
|
+
vectors = self._vectors_for_rows(active, texts)
|
|
536
|
+
kept_rows = []
|
|
537
|
+
kept_vectors = []
|
|
538
|
+
for row in rows:
|
|
539
|
+
vector = vectors.get(row[0])
|
|
540
|
+
if vector is None:
|
|
541
|
+
continue
|
|
542
|
+
if kept_vectors and len(vector) != len(kept_vectors[0]):
|
|
543
|
+
continue
|
|
544
|
+
kept_rows.append(row)
|
|
545
|
+
kept_vectors.append(vector)
|
|
546
|
+
matrix = None
|
|
547
|
+
if np is not None and kept_vectors:
|
|
548
|
+
matrix = np.asarray(kept_vectors, dtype=np.float32)
|
|
549
|
+
matrix = _normalized_rows(matrix)
|
|
550
|
+
mode = self._resolved_vector_mode(len(kept_rows))
|
|
551
|
+
ann = None
|
|
552
|
+
ann_build_s = 0.0
|
|
553
|
+
if mode == "ann" and matrix is not None:
|
|
554
|
+
ann_started = time.perf_counter()
|
|
555
|
+
try:
|
|
556
|
+
ann = build_ivf(matrix)
|
|
557
|
+
except (MemoryError, RuntimeError, ValueError):
|
|
558
|
+
ann = None
|
|
559
|
+
ann_build_s = time.perf_counter() - ann_started
|
|
560
|
+
if ann is None:
|
|
561
|
+
mode = "exact"
|
|
562
|
+
rowids = [row[0] for row in kept_rows]
|
|
563
|
+
self._vector_index = {
|
|
564
|
+
"key": key,
|
|
565
|
+
"rows": kept_rows,
|
|
566
|
+
"rowids": rowids,
|
|
567
|
+
"matrix": matrix,
|
|
568
|
+
"positions": {rowid: index
|
|
569
|
+
for index, rowid in enumerate(rowids)},
|
|
570
|
+
"mode": mode,
|
|
571
|
+
"ann": ann,
|
|
572
|
+
"ann_build_s": ann_build_s,
|
|
573
|
+
"build_s": time.perf_counter() - started,
|
|
574
|
+
}
|
|
575
|
+
self._vector_fallback_count = 0
|
|
576
|
+
return self.vector_index_stats()
|
|
577
|
+
|
|
578
|
+
def _resolved_vector_mode(self, chunks: int) -> str:
|
|
579
|
+
if self.vector_index_mode == "exact" or np is None:
|
|
580
|
+
return "exact"
|
|
581
|
+
if self.vector_index_mode == "ann":
|
|
582
|
+
return "ann" if chunks >= 2 else "exact"
|
|
583
|
+
return ("ann" if chunks >= max(2, int(self.ann_min_chunks))
|
|
584
|
+
else "exact")
|
|
585
|
+
|
|
586
|
+
def vector_index_stats(self) -> dict:
|
|
587
|
+
index = self._vector_index
|
|
588
|
+
backend = "numpy" if np is not None else "scan"
|
|
589
|
+
if not index:
|
|
590
|
+
return {"built": False, "chunks": 0, "dim": 0,
|
|
591
|
+
"build_s": 0.0, "backend": backend}
|
|
592
|
+
matrix = index.get("matrix")
|
|
593
|
+
ann = index.get("ann")
|
|
594
|
+
nlist = int(ann["nlist"]) if ann else 0
|
|
595
|
+
return {
|
|
596
|
+
"built": matrix is not None,
|
|
597
|
+
"chunks": len(index.get("rows", [])),
|
|
598
|
+
"dim": int(matrix.shape[1]) if matrix is not None else 0,
|
|
599
|
+
"mode": index.get("mode", "exact"),
|
|
600
|
+
"ann": bool(ann),
|
|
601
|
+
"nlist": nlist,
|
|
602
|
+
"nprobe": (int(self.ann_probe) if self.ann_probe
|
|
603
|
+
else (max(1, int(math.ceil(math.sqrt(nlist))))
|
|
604
|
+
if nlist else 0)),
|
|
605
|
+
"exact_fallback": True,
|
|
606
|
+
"fallbacks": int(self._vector_fallback_count),
|
|
607
|
+
"build_s": round(float(index.get("build_s", 0.0)), 3),
|
|
608
|
+
"ann_build_s": round(float(index.get("ann_build_s", 0.0)), 3),
|
|
609
|
+
"backend": backend,
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
@staticmethod
|
|
613
|
+
def _candidate_filters(query: str, neural: bool):
|
|
614
|
+
threshold = HYBRID_MIN_NEURAL if neural else HYBRID_MIN_HASHED
|
|
615
|
+
grams = set(_char_ngrams(query)) if not neural else None
|
|
616
|
+
return threshold, grams
|
|
617
|
+
|
|
618
|
+
@staticmethod
|
|
619
|
+
def _has_hashed_overlap(query_grams, row) -> bool:
|
|
620
|
+
if query_grams is None:
|
|
621
|
+
return True
|
|
622
|
+
# Hashed vectors are sparse and can collide on short queries. Require
|
|
623
|
+
# real character overlap before a hash-only candidate may bypass FTS.
|
|
624
|
+
text = f"{row[3]} {row[1]}".lower()
|
|
625
|
+
return len(query_grams & set(_char_ngrams(text))) >= 2
|
|
626
|
+
|
|
627
|
+
@staticmethod
|
|
628
|
+
def _exact_candidates(index, query_vector, limit: int):
|
|
629
|
+
scores = index["matrix"] @ query_vector
|
|
630
|
+
order = np.argsort(-scores, kind="stable")[:limit]
|
|
631
|
+
return order, scores[order]
|
|
632
|
+
|
|
633
|
+
def _search_positions(self, index, query_vector, limit: int,
|
|
634
|
+
required_rowids, threshold: float):
|
|
635
|
+
matrix = index["matrix"]
|
|
636
|
+
ann = index.get("ann")
|
|
637
|
+
positions = None
|
|
638
|
+
scores = None
|
|
639
|
+
if ann is not None:
|
|
640
|
+
try:
|
|
641
|
+
positions, scores = ann_topk(
|
|
642
|
+
matrix, ann, query_vector, limit, self.ann_probe)
|
|
643
|
+
if not len(positions) or float(scores[0]) < threshold:
|
|
644
|
+
raise _AnnFallback("low ANN confidence")
|
|
645
|
+
except Exception:
|
|
646
|
+
positions = None
|
|
647
|
+
scores = None
|
|
648
|
+
if positions is None:
|
|
649
|
+
if ann is not None:
|
|
650
|
+
self._vector_fallback_count += 1
|
|
651
|
+
positions, scores = self._exact_candidates(
|
|
652
|
+
index, query_vector, limit)
|
|
653
|
+
by_rowid = {
|
|
654
|
+
index["rowids"][int(position)]: float(score)
|
|
655
|
+
for position, score in zip(positions, scores)
|
|
656
|
+
}
|
|
657
|
+
required = [index["positions"][rowid]
|
|
658
|
+
for rowid in required_rowids or ()
|
|
659
|
+
if rowid in index["positions"]]
|
|
660
|
+
if required:
|
|
661
|
+
extra = matrix[required] @ query_vector
|
|
662
|
+
for position, score in zip(required, extra):
|
|
663
|
+
by_rowid[index["rowids"][position]] = float(score)
|
|
664
|
+
return ([int(position) for position in positions],
|
|
665
|
+
[float(score) for score in scores], by_rowid)
|
|
666
|
+
|
|
667
|
+
def _embedding_candidates(self, query: str, active, limit: int,
|
|
668
|
+
required_rowids=None):
|
|
669
|
+
"""Score chunks with the prewarmed matrix; rebuild it when invalid."""
|
|
670
|
+
key = self._vector_model_key(active)
|
|
671
|
+
index = self._vector_index
|
|
672
|
+
if index is None or index.get("key") != key:
|
|
673
|
+
self.prewarm_vectors(active)
|
|
674
|
+
index = self._vector_index
|
|
675
|
+
matrix = index.get("matrix") if index else None
|
|
676
|
+
if matrix is None:
|
|
677
|
+
return self._embedding_candidates_scan(
|
|
678
|
+
query, active, limit, required_rowids)
|
|
679
|
+
if not index["rows"]:
|
|
680
|
+
return [], {}
|
|
681
|
+
query_vector = np.asarray(
|
|
682
|
+
self._embed_texts(active, [query])[0], dtype=np.float32)
|
|
683
|
+
if matrix.shape[1] != query_vector.shape[0]:
|
|
684
|
+
return self._embedding_candidates_scan(
|
|
685
|
+
query, active, limit, required_rowids)
|
|
686
|
+
neural = key.startswith("neural:")
|
|
687
|
+
threshold, query_grams = self._candidate_filters(query, neural)
|
|
688
|
+
try:
|
|
689
|
+
top, top_scores, by_rowid = self._search_positions(
|
|
690
|
+
index, query_vector, limit, required_rowids, threshold)
|
|
691
|
+
except (MemoryError, RuntimeError, ValueError, IndexError):
|
|
692
|
+
return self._embedding_candidates_scan(
|
|
693
|
+
query, active, limit, required_rowids)
|
|
694
|
+
hits = []
|
|
695
|
+
for position, similarity in zip(top, top_scores):
|
|
696
|
+
if similarity < threshold:
|
|
697
|
+
break
|
|
698
|
+
row = index["rows"][position]
|
|
699
|
+
if not self._has_hashed_overlap(query_grams, row):
|
|
700
|
+
continue
|
|
701
|
+
hit = {
|
|
702
|
+
"chunk": row[1], "source": row[2], "title": row[3],
|
|
703
|
+
"score": None, "_rowid": row[0],
|
|
704
|
+
"embedding_score": similarity,
|
|
705
|
+
}
|
|
706
|
+
if neural:
|
|
707
|
+
hit["neural_score"] = similarity
|
|
708
|
+
hits.append(hit)
|
|
709
|
+
return hits, by_rowid
|
|
710
|
+
|
|
711
|
+
def _embedding_candidates_scan(self, query: str, active, limit: int,
|
|
712
|
+
required_rowids=None):
|
|
713
|
+
"""Legacy linear scan; used only when numpy is unavailable."""
|
|
714
|
+
rows = self.con.execute(
|
|
715
|
+
"SELECT rowid, chunk, source, title FROM docs "
|
|
716
|
+
"ORDER BY rowid").fetchall()
|
|
717
|
+
if not rows:
|
|
718
|
+
return [], {}
|
|
719
|
+
texts = [(row[0], f"{row[3]} {row[1]}".strip()) for row in rows]
|
|
720
|
+
vectors = self._vectors_for_rows(active, texts)
|
|
721
|
+
query_vector = self._embed_texts(active, [query])[0]
|
|
722
|
+
neural = self._vector_model_key(active).startswith("neural:")
|
|
723
|
+
scored = []
|
|
724
|
+
for row in rows:
|
|
725
|
+
vector = vectors.get(row[0])
|
|
726
|
+
if vector is None or len(vector) != len(query_vector):
|
|
727
|
+
continue
|
|
728
|
+
similarity = sum(a * b for a, b in
|
|
729
|
+
zip(query_vector, vector))
|
|
730
|
+
scored.append((similarity, row))
|
|
731
|
+
scored.sort(key=lambda item: (-item[0], item[1][0]))
|
|
732
|
+
by_rowid = {row[0]: similarity for similarity, row in scored}
|
|
733
|
+
threshold, query_grams = self._candidate_filters(query, neural)
|
|
734
|
+
hits = []
|
|
735
|
+
for similarity, row in scored[:limit]:
|
|
736
|
+
if similarity < threshold:
|
|
737
|
+
break
|
|
738
|
+
if not self._has_hashed_overlap(query_grams, row):
|
|
739
|
+
continue
|
|
740
|
+
hit = {
|
|
741
|
+
"chunk": row[1], "source": row[2], "title": row[3],
|
|
742
|
+
"score": None, "_rowid": row[0],
|
|
743
|
+
"embedding_score": similarity,
|
|
744
|
+
}
|
|
745
|
+
if neural:
|
|
746
|
+
hit["neural_score"] = similarity
|
|
747
|
+
hits.append(hit)
|
|
748
|
+
return hits, by_rowid
|
|
749
|
+
|
|
750
|
+
def _meta_map(self, sources) -> dict[str, dict]:
|
|
751
|
+
sources = list(sources)
|
|
752
|
+
if not sources:
|
|
753
|
+
return {}
|
|
754
|
+
placeholders = ",".join("?" for _ in sources)
|
|
755
|
+
rows = self.con.execute(
|
|
756
|
+
"SELECT source, kind, source_url, teacher, model, verified, "
|
|
757
|
+
f"created, reviewer, verified_at FROM doc_meta "
|
|
758
|
+
f"WHERE source IN ({placeholders})",
|
|
759
|
+
tuple(sources)).fetchall()
|
|
760
|
+
return {row[0]: {
|
|
761
|
+
"source": row[0],
|
|
762
|
+
"kind": row[1],
|
|
763
|
+
"source_url": row[2],
|
|
764
|
+
"teacher": row[3],
|
|
765
|
+
"model": row[4],
|
|
766
|
+
"verified": bool(row[5]),
|
|
767
|
+
"created": row[6],
|
|
768
|
+
"reviewer": row[7],
|
|
769
|
+
"verified_at": row[8],
|
|
770
|
+
} for row in rows}
|
|
771
|
+
|
|
772
|
+
def _chunk_meta_map(self, rowids) -> dict[int, dict]:
|
|
773
|
+
rowids = [int(rowid) for rowid in rowids
|
|
774
|
+
if rowid is not None]
|
|
775
|
+
if not rowids:
|
|
776
|
+
return {}
|
|
777
|
+
placeholders = ",".join("?" for _ in rowids)
|
|
778
|
+
rows = self.con.execute(
|
|
779
|
+
"SELECT doc_rowid, page, section, char_start, char_end "
|
|
780
|
+
f"FROM doc_chunk_meta WHERE doc_rowid IN ({placeholders})",
|
|
781
|
+
tuple(rowids)).fetchall()
|
|
782
|
+
return {row[0]: {
|
|
783
|
+
"page": row[1],
|
|
784
|
+
"section": row[2] or "",
|
|
785
|
+
"char_start": row[3],
|
|
786
|
+
"char_end": row[4],
|
|
787
|
+
} for row in rows}
|
|
788
|
+
|
|
789
|
+
def search(self, query: str, k: int = 5,
|
|
790
|
+
rerank: bool = True,
|
|
791
|
+
reranker=None) -> list[dict]:
|
|
792
|
+
words = re.findall(r"[\wçğıöşüÇĞİÖŞÜ]+", query)
|
|
793
|
+
words = [w for w in words if len(w) > 1]
|
|
794
|
+
meaningful = [w for w in words if w.lower() not in STOPWORDS]
|
|
795
|
+
if meaningful:
|
|
796
|
+
words = meaningful
|
|
797
|
+
if not words:
|
|
798
|
+
return []
|
|
799
|
+
match = " OR ".join(f'"{w}"' for w in words[:12])
|
|
800
|
+
candidates = max(k * 4, 12) if rerank else k
|
|
801
|
+
active = reranker if reranker is not None else self.reranker
|
|
802
|
+
neural = self._vector_model_key(active).startswith("neural:")
|
|
803
|
+
with self._lock:
|
|
804
|
+
rows = self.con.execute(
|
|
805
|
+
"SELECT rowid, chunk, source, title, bm25(docs) AS score "
|
|
806
|
+
"FROM docs WHERE docs MATCH ? ORDER BY score LIMIT ?",
|
|
807
|
+
(match, candidates)).fetchall()
|
|
808
|
+
embedding_hits, embedding_scores = ([], {})
|
|
809
|
+
# Hashed n-grams are a fallback reranker, not a semantic retriever:
|
|
810
|
+
# do not answer from hash-only candidates when FTS found nothing.
|
|
811
|
+
# The neural reranker has a calibrated threshold and may do so.
|
|
812
|
+
if rerank and (rows or neural):
|
|
813
|
+
embedding_hits, embedding_scores = \
|
|
814
|
+
self._embedding_candidates(
|
|
815
|
+
query, active, candidates,
|
|
816
|
+
required_rowids={row[0] for row in rows})
|
|
817
|
+
sources = {row[2] for row in rows} | {
|
|
818
|
+
hit["source"] for hit in embedding_hits}
|
|
819
|
+
metas = self._meta_map(sources)
|
|
820
|
+
chunk_metas = self._chunk_meta_map(
|
|
821
|
+
{row[0] for row in rows}
|
|
822
|
+
| {hit.get("_rowid") for hit in embedding_hits})
|
|
823
|
+
default = {"kind": "document", "source_url": "", "teacher": "",
|
|
824
|
+
"model": "", "verified": True, "created": "",
|
|
825
|
+
"reviewer": "", "verified_at": "", "page": None,
|
|
826
|
+
"section": "", "char_start": None, "char_end": None}
|
|
827
|
+
hits = [{
|
|
828
|
+
"chunk": row[1],
|
|
829
|
+
"source": row[2],
|
|
830
|
+
"title": row[3],
|
|
831
|
+
"score": float(row[4]),
|
|
832
|
+
"_rowid": row[0],
|
|
833
|
+
**default,
|
|
834
|
+
**metas.get(row[2], {}),
|
|
835
|
+
**chunk_metas.get(row[0], {}),
|
|
836
|
+
} for row in rows]
|
|
837
|
+
for hit in embedding_hits:
|
|
838
|
+
hit.update(chunk_metas.get(hit.get("_rowid"), {}))
|
|
839
|
+
if not rerank:
|
|
840
|
+
return hits
|
|
841
|
+
# Hybrid candidate pool: FTS rows plus the best embedding rows. A
|
|
842
|
+
# paraphrase with no lexical overlap with any document can now enter
|
|
843
|
+
# the reranker, which was previously capped by FTS recall.
|
|
844
|
+
merged = []
|
|
845
|
+
seen = set()
|
|
846
|
+
for hit in hits + embedding_hits:
|
|
847
|
+
key = (hit["source"], hit["chunk"])
|
|
848
|
+
if key in seen:
|
|
849
|
+
continue
|
|
850
|
+
seen.add(key)
|
|
851
|
+
merged.append(hit)
|
|
852
|
+
finite = [hit["score"] for hit in hits
|
|
853
|
+
if hit.get("score") is not None]
|
|
854
|
+
fallback = max(finite) if finite else None
|
|
855
|
+
for hit in merged:
|
|
856
|
+
similarity = embedding_scores.get(hit.get("_rowid"))
|
|
857
|
+
if similarity is not None:
|
|
858
|
+
hit["embedding_score"] = similarity
|
|
859
|
+
if neural:
|
|
860
|
+
hit["neural_score"] = similarity
|
|
861
|
+
if hit.get("score") is None:
|
|
862
|
+
hit["score"] = fallback
|
|
863
|
+
if active is not None:
|
|
864
|
+
ranked = active.rerank(query, merged, k)
|
|
865
|
+
else:
|
|
866
|
+
ranked = rerank_hits(query, merged, k)
|
|
867
|
+
for hit in ranked:
|
|
868
|
+
hit.pop("_rowid", None)
|
|
869
|
+
return ranked
|
|
870
|
+
|
|
871
|
+
def sources(self) -> list[dict]:
|
|
872
|
+
with self._lock:
|
|
873
|
+
rows = self.con.execute(
|
|
874
|
+
"SELECT source, title, COUNT(*), MAX(sha) FROM docs "
|
|
875
|
+
"GROUP BY source, title ORDER BY source").fetchall()
|
|
876
|
+
metas = self._meta_map([row[0] for row in rows])
|
|
877
|
+
page_rows = self.con.execute(
|
|
878
|
+
"SELECT docs.source, MIN(m.page), MAX(m.page) "
|
|
879
|
+
"FROM doc_chunk_meta m JOIN docs ON docs.rowid = m.doc_rowid "
|
|
880
|
+
"GROUP BY docs.source").fetchall()
|
|
881
|
+
pages = {row[0]: (row[1], row[2]) for row in page_rows}
|
|
882
|
+
result = []
|
|
883
|
+
for row in rows:
|
|
884
|
+
item = {
|
|
885
|
+
"source": row[0],
|
|
886
|
+
"title": row[1],
|
|
887
|
+
"chunks": row[2],
|
|
888
|
+
"content_sha256": row[3],
|
|
889
|
+
**metas.get(row[0], {"source": row[0], "kind": "document",
|
|
890
|
+
"source_url": "", "teacher": "",
|
|
891
|
+
"model": "", "verified": True,
|
|
892
|
+
"created": "", "reviewer": "",
|
|
893
|
+
"verified_at": ""}),
|
|
894
|
+
}
|
|
895
|
+
page_range = pages.get(row[0])
|
|
896
|
+
if page_range and page_range[0] is not None:
|
|
897
|
+
item["page_start"] = page_range[0]
|
|
898
|
+
item["page_end"] = page_range[1]
|
|
899
|
+
result.append(item)
|
|
900
|
+
return result
|
|
901
|
+
|
|
902
|
+
def source_evidence(self, source: str, page: int | None = None,
|
|
903
|
+
section: str = "", limit: int = 20) -> dict:
|
|
904
|
+
"""Return the stored chunks for a source/page/section locator."""
|
|
905
|
+
source = str(source or "").strip()
|
|
906
|
+
if not source:
|
|
907
|
+
raise ValueError("source is required")
|
|
908
|
+
if len(source) > 512:
|
|
909
|
+
raise ValueError("source is too long (max 512 chars)")
|
|
910
|
+
if any(ord(ch) < 32 for ch in source):
|
|
911
|
+
raise ValueError("source contains control characters")
|
|
912
|
+
try:
|
|
913
|
+
limit = int(limit)
|
|
914
|
+
except (TypeError, ValueError) as exc:
|
|
915
|
+
raise ValueError("limit must be an integer") from exc
|
|
916
|
+
limit = max(1, min(limit, 50))
|
|
917
|
+
clauses = ["d.source = ?"]
|
|
918
|
+
params: list = [source]
|
|
919
|
+
if page is not None:
|
|
920
|
+
try:
|
|
921
|
+
page = int(page)
|
|
922
|
+
except (TypeError, ValueError) as exc:
|
|
923
|
+
raise ValueError("page must be an integer") from exc
|
|
924
|
+
if page < 1:
|
|
925
|
+
raise ValueError("page must be >= 1")
|
|
926
|
+
clauses.append("m.page = ?")
|
|
927
|
+
params.append(page)
|
|
928
|
+
section = str(section or "").strip()
|
|
929
|
+
if section:
|
|
930
|
+
if len(section) > 200:
|
|
931
|
+
raise ValueError("section is too long (max 200 chars)")
|
|
932
|
+
clauses.append("lower(m.section) = lower(?)")
|
|
933
|
+
params.append(section)
|
|
934
|
+
with self._lock:
|
|
935
|
+
rows = self.con.execute(
|
|
936
|
+
"SELECT d.chunk, d.source, d.title, m.page, m.section, "
|
|
937
|
+
"m.char_start, m.char_end FROM docs d "
|
|
938
|
+
"LEFT JOIN doc_chunk_meta m ON m.doc_rowid = d.rowid "
|
|
939
|
+
f"WHERE {' AND '.join(clauses)} "
|
|
940
|
+
"ORDER BY COALESCE(m.page, 0), d.rowid LIMIT ?",
|
|
941
|
+
(*params, limit)).fetchall()
|
|
942
|
+
if not rows:
|
|
943
|
+
raise SourceNotFound(
|
|
944
|
+
f"no evidence for source/page/section: {source}")
|
|
945
|
+
data = [{
|
|
946
|
+
"chunk": row[0],
|
|
947
|
+
"source": row[1],
|
|
948
|
+
"title": row[2],
|
|
949
|
+
"page": row[3],
|
|
950
|
+
"section": row[4] or "",
|
|
951
|
+
"char_start": row[5],
|
|
952
|
+
"char_end": row[6],
|
|
953
|
+
} for row in rows]
|
|
954
|
+
return {
|
|
955
|
+
"source": source,
|
|
956
|
+
"page": page,
|
|
957
|
+
"section": section,
|
|
958
|
+
"returned": len(data),
|
|
959
|
+
"data": data,
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
def verify_source(self, source: str, reviewer: str) -> dict:
|
|
963
|
+
"""Mark a teacher note verified with reviewer and timestamp."""
|
|
964
|
+
source = str(source or "").strip()
|
|
965
|
+
reviewer = str(reviewer or "").strip()
|
|
966
|
+
if not source:
|
|
967
|
+
raise ValueError("source is required")
|
|
968
|
+
if not reviewer:
|
|
969
|
+
raise ValueError("reviewer is required")
|
|
970
|
+
if len(reviewer) > 100:
|
|
971
|
+
raise ValueError("reviewer is too long (max 100 chars)")
|
|
972
|
+
if any(ord(ch) < 32 for ch in reviewer):
|
|
973
|
+
raise ValueError("reviewer contains control characters")
|
|
974
|
+
with self._lock:
|
|
975
|
+
row = self.con.execute(
|
|
976
|
+
"SELECT kind FROM doc_meta WHERE source = ?",
|
|
977
|
+
(source,)).fetchone()
|
|
978
|
+
if row is None:
|
|
979
|
+
raise SourceNotFound(f"source not found: {source}")
|
|
980
|
+
if row[0] != "teacher":
|
|
981
|
+
raise ValueError("only teacher notes can be verified")
|
|
982
|
+
verified_at = time.strftime("%Y-%m-%dT%H:%M:%S")
|
|
983
|
+
self.con.execute(
|
|
984
|
+
"UPDATE doc_meta SET verified = 1, reviewer = ?, "
|
|
985
|
+
"verified_at = ? WHERE source = ?",
|
|
986
|
+
(reviewer, verified_at, source))
|
|
987
|
+
self.con.commit()
|
|
988
|
+
meta = self._meta_map([source]).get(source, {})
|
|
989
|
+
return {"source": source, **meta}
|
|
990
|
+
|
|
991
|
+
def forget(self, source: str) -> dict:
|
|
992
|
+
"""Delete one source and its metadata, then compact the database."""
|
|
993
|
+
source = str(source or "").strip()
|
|
994
|
+
if not source:
|
|
995
|
+
raise ValueError("source is required")
|
|
996
|
+
variants = [source]
|
|
997
|
+
if "://" not in source:
|
|
998
|
+
# Windows stores local paths with backslashes; accept either form
|
|
999
|
+
# on every platform (Path() treats a backslash as a normal char on
|
|
1000
|
+
# POSIX, so do the replacement explicitly too).
|
|
1001
|
+
for variant in (source.replace("/", "\\"),
|
|
1002
|
+
source.replace("\\", "/"),
|
|
1003
|
+
Path(source).as_posix(), str(Path(source))):
|
|
1004
|
+
if variant not in variants:
|
|
1005
|
+
variants.append(variant)
|
|
1006
|
+
placeholders = ",".join("?" for _ in variants)
|
|
1007
|
+
with self._lock:
|
|
1008
|
+
chunks = self.con.execute(
|
|
1009
|
+
f"SELECT COUNT(*) FROM docs WHERE source IN ({placeholders})",
|
|
1010
|
+
tuple(variants)).fetchone()[0]
|
|
1011
|
+
meta = self.con.execute(
|
|
1012
|
+
f"SELECT COUNT(*) FROM doc_meta WHERE source IN "
|
|
1013
|
+
f"({placeholders})", tuple(variants)).fetchone()[0]
|
|
1014
|
+
if chunks == 0 and meta == 0:
|
|
1015
|
+
raise SourceNotFound(f"source not found: {source}")
|
|
1016
|
+
self.con.execute(
|
|
1017
|
+
f"DELETE FROM docs WHERE source IN ({placeholders})",
|
|
1018
|
+
tuple(variants))
|
|
1019
|
+
self.con.execute(
|
|
1020
|
+
f"DELETE FROM doc_meta WHERE source IN ({placeholders})",
|
|
1021
|
+
tuple(variants))
|
|
1022
|
+
self._drop_stale_vectors()
|
|
1023
|
+
self._drop_stale_chunk_meta()
|
|
1024
|
+
self._vector_index = None
|
|
1025
|
+
self.con.commit()
|
|
1026
|
+
vacuumed = False
|
|
1027
|
+
try:
|
|
1028
|
+
self.con.execute("VACUUM")
|
|
1029
|
+
vacuumed = True
|
|
1030
|
+
except sqlite3.OperationalError:
|
|
1031
|
+
pass
|
|
1032
|
+
return {"source": source, "chunks_deleted": int(chunks),
|
|
1033
|
+
"meta_deleted": int(meta), "vacuumed": vacuumed}
|
|
1034
|
+
|
|
1035
|
+
def forget_all(self) -> dict:
|
|
1036
|
+
"""Delete every indexed source and compact the database."""
|
|
1037
|
+
with self._lock:
|
|
1038
|
+
chunks = self.con.execute(
|
|
1039
|
+
"SELECT COUNT(*) FROM docs").fetchone()[0]
|
|
1040
|
+
meta = self.con.execute(
|
|
1041
|
+
"SELECT COUNT(*) FROM doc_meta").fetchone()[0]
|
|
1042
|
+
self.con.execute("DELETE FROM docs")
|
|
1043
|
+
self.con.execute("DELETE FROM doc_meta")
|
|
1044
|
+
self._drop_stale_vectors()
|
|
1045
|
+
self._drop_stale_chunk_meta()
|
|
1046
|
+
self._vector_index = None
|
|
1047
|
+
self.con.commit()
|
|
1048
|
+
vacuumed = False
|
|
1049
|
+
try:
|
|
1050
|
+
self.con.execute("VACUUM")
|
|
1051
|
+
vacuumed = True
|
|
1052
|
+
except sqlite3.OperationalError:
|
|
1053
|
+
pass
|
|
1054
|
+
return {"chunks_deleted": int(chunks),
|
|
1055
|
+
"meta_deleted": int(meta), "vacuumed": vacuumed}
|
|
1056
|
+
|
|
1057
|
+
def stats(self) -> dict:
|
|
1058
|
+
with self._lock:
|
|
1059
|
+
count = self.con.execute(
|
|
1060
|
+
"SELECT COUNT(*) FROM docs").fetchone()[0]
|
|
1061
|
+
return {"path": str(self.path), "chunks": count,
|
|
1062
|
+
"sources": len(self.sources())}
|
|
1063
|
+
|
|
1064
|
+
def close(self):
|
|
1065
|
+
with self._lock:
|
|
1066
|
+
self.con.close()
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def main(argv=None):
|
|
1070
|
+
ap = argparse.ArgumentParser()
|
|
1071
|
+
ap.add_argument("--db", default=None)
|
|
1072
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
1073
|
+
add = sub.add_parser("add")
|
|
1074
|
+
add.add_argument("path")
|
|
1075
|
+
add.add_argument("--title", default="")
|
|
1076
|
+
search = sub.add_parser("search")
|
|
1077
|
+
search.add_argument("query")
|
|
1078
|
+
search.add_argument("-k", type=int, default=5)
|
|
1079
|
+
list_db = sub.add_parser("list")
|
|
1080
|
+
stats_db = sub.add_parser("stats")
|
|
1081
|
+
forget = sub.add_parser("forget")
|
|
1082
|
+
forget.add_argument("source", nargs="?")
|
|
1083
|
+
forget.add_argument("--all", action="store_true")
|
|
1084
|
+
# Accept --db on either side of the subcommand; the docs use both forms.
|
|
1085
|
+
for parser in (add, search, list_db, stats_db, forget):
|
|
1086
|
+
parser.add_argument("--db", dest="db_after", default=None)
|
|
1087
|
+
args = ap.parse_args(argv)
|
|
1088
|
+
mem = DocMemory(args.db_after or args.db or "data/docs.db")
|
|
1089
|
+
if args.cmd == "add":
|
|
1090
|
+
print(json.dumps({"added": mem.add_file(args.path, args.title)}))
|
|
1091
|
+
elif args.cmd == "search":
|
|
1092
|
+
print(json.dumps(mem.search(args.query, args.k), indent=2,
|
|
1093
|
+
ensure_ascii=False))
|
|
1094
|
+
elif args.cmd == "list":
|
|
1095
|
+
print(json.dumps(mem.sources(), indent=2, ensure_ascii=False))
|
|
1096
|
+
elif args.cmd == "forget":
|
|
1097
|
+
try:
|
|
1098
|
+
result = (mem.forget_all() if args.all
|
|
1099
|
+
else mem.forget(args.source or ""))
|
|
1100
|
+
except (ValueError, SourceNotFound) as exc:
|
|
1101
|
+
print(json.dumps({"error": str(exc)}))
|
|
1102
|
+
mem.close()
|
|
1103
|
+
return 1
|
|
1104
|
+
print(json.dumps(result, ensure_ascii=False))
|
|
1105
|
+
else:
|
|
1106
|
+
print(json.dumps(mem.stats(), indent=2))
|
|
1107
|
+
mem.close()
|
|
1108
|
+
return 0
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
if __name__ == "__main__":
|
|
1112
|
+
raise SystemExit(main())
|