atomzero 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
atomzero/doc_memory.py ADDED
@@ -0,0 +1,1112 @@
1
+ """Local document memory with SQLite FTS5 and provenance.
2
+
3
+ Documents are chunked, indexed and retrieved with their source. Facts stay in
4
+ this index, not in the model weights.
5
+
6
+ python -m atomzero.doc_memory add notes.txt --db data/docs.db
7
+ python -m atomzero.doc_memory --db data/docs.db search "kira"
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import array
13
+ import hashlib
14
+ import json
15
+ import math
16
+ import re
17
+ import sqlite3
18
+ import threading
19
+ import time
20
+ import zlib
21
+ from pathlib import Path
22
+
23
+ try:
24
+ import numpy as np
25
+ except ImportError: # pragma: no cover - numpy is a declared dependency
26
+ np = None
27
+
28
+ MAX_CHARS = 900
29
+
30
+
31
+ class SourceNotFound(ValueError):
32
+ pass
33
+
34
+
35
+ STOPWORDS = frozenset({
36
+ "a", "an", "and", "are", "as", "at", "be", "been", "being", "but",
37
+ "by", "can", "could", "did", "do", "does", "for", "from", "had", "has",
38
+ "have", "he", "her", "here", "him", "his", "how", "i", "if", "in",
39
+ "into", "is", "it", "its", "me", "my", "no", "not", "of", "on", "or",
40
+ "our", "please", "she", "should", "so", "tell", "than", "that", "the",
41
+ "their", "them", "then", "there", "these", "they", "this", "those",
42
+ "to", "was", "we", "were", "what", "when", "where", "which", "who",
43
+ "why", "will", "with", "would", "you", "your",
44
+ })
45
+
46
+
47
+ def _split_long(paragraph: str) -> list[str]:
48
+ """Split a long paragraph near a word boundary, never mid-word if possible."""
49
+ out = []
50
+ while len(paragraph) > MAX_CHARS:
51
+ cut = paragraph.rfind(" ", 0, MAX_CHARS)
52
+ if cut < MAX_CHARS // 2:
53
+ cut = MAX_CHARS
54
+ piece = paragraph[:cut].strip()
55
+ if piece:
56
+ out.append(piece)
57
+ paragraph = paragraph[cut:].lstrip()
58
+ if paragraph.strip():
59
+ out.append(paragraph.strip())
60
+ return out
61
+
62
+
63
+ EMBED_DIM = 512
64
+ HASHED_CACHE_KEY = "hashed-ngram-v1"
65
+ # Minimum cosine for a semantic-only candidate. Lexical FTS hits are always
66
+ # kept; this only stops unrelated chunks from being dragged into the pool.
67
+ HYBRID_MIN_HASHED = 0.28
68
+ HYBRID_MIN_NEURAL = 0.30
69
+ ANN_MIN_CHUNKS = 256
70
+ ANN_BUILD_ITERATIONS = 8
71
+
72
+
73
+ class _AnnFallback(RuntimeError):
74
+ """Internal signal: use the exact matrix for this query."""
75
+
76
+
77
+ def _char_ngrams(text: str) -> list[str]:
78
+ words = re.findall(r"[^\W_]+", str(text).lower(), re.UNICODE)
79
+ grams = []
80
+ for word in words:
81
+ padded = f" {word} "
82
+ for size in (3, 4, 5):
83
+ grams.extend(padded[index:index + size]
84
+ for index in range(len(padded) - size + 1))
85
+ return grams
86
+
87
+
88
+ def embed_text(text: str, dim: int = EMBED_DIM) -> list[float]:
89
+ """Deterministic hashed character n-gram embedding (no neural model)."""
90
+ vector = [0.0] * dim
91
+ for gram in _char_ngrams(text):
92
+ vector[zlib.crc32(gram.encode("utf-8")) % dim] += 1.0
93
+ norm = math.sqrt(sum(value * value for value in vector))
94
+ if norm:
95
+ vector = [value / norm for value in vector]
96
+ return vector
97
+
98
+
99
+ def embedding_similarity(left: str, right: str) -> float:
100
+ a, b = embed_text(left), embed_text(right)
101
+ return sum(x * y for x, y in zip(a, b))
102
+
103
+
104
+ def _query_terms(text: str) -> set[str]:
105
+ words = re.findall(r"[^\W_]+", str(text).lower(), re.UNICODE)
106
+ return {word for word in words
107
+ if len(word) > 2 and word not in STOPWORDS}
108
+
109
+
110
+ def _pack_vector(values) -> bytes:
111
+ return array.array("f", (float(value) for value in values)).tobytes()
112
+
113
+
114
+ def _unpack_vector(blob: bytes):
115
+ values = array.array("f")
116
+ values.frombytes(blob)
117
+ return values
118
+
119
+
120
+ def _normalized_rows(matrix):
121
+ norms = np.linalg.norm(matrix, axis=1, keepdims=True)
122
+ norms[norms == 0.0] = 1.0
123
+ return matrix / norms
124
+
125
+
126
+ def build_ivf(matrix, nlist: int | None = None,
127
+ iterations: int = ANN_BUILD_ITERATIONS) -> dict | None:
128
+ """Deterministic k-means IVF index over normalized rows.
129
+
130
+ This is approximate retrieval: a query scores cluster centroids, probes
131
+ the best `nprobe` buckets, then reranks only their members exactly.
132
+ """
133
+ if np is None or matrix is None or len(matrix) < 2:
134
+ return None
135
+ n = len(matrix)
136
+ if nlist is None:
137
+ nlist = min(64, max(2, int(round(math.sqrt(n)))))
138
+ nlist = min(max(2, int(nlist)), n)
139
+ starts = np.linspace(0, n - 1, nlist).astype(int)
140
+ centroids = matrix[starts].copy()
141
+ for _ in range(max(1, iterations)):
142
+ assignments = (matrix @ centroids.T).argmax(axis=1)
143
+ updated = centroids.copy()
144
+ for cluster in range(nlist):
145
+ members = matrix[assignments == cluster]
146
+ if len(members):
147
+ updated[cluster] = members.mean(axis=0)
148
+ centroids = _normalized_rows(updated)
149
+ assignments = (matrix @ centroids.T).argmax(axis=1)
150
+ buckets = [np.where(assignments == cluster)[0].astype(np.int32)
151
+ for cluster in range(nlist)]
152
+ return {"centroids": centroids, "buckets": buckets, "nlist": nlist}
153
+
154
+
155
+ def ann_topk(matrix, ann: dict, query_vector, limit: int,
156
+ probe: int | None = None):
157
+ """Return exact-scored top-k within the best `probe` IVF buckets."""
158
+ if ann is None or matrix is None:
159
+ raise _AnnFallback("no ANN index")
160
+ nlist = int(ann["nlist"])
161
+ if probe is None:
162
+ probe = max(1, int(math.ceil(math.sqrt(nlist))))
163
+ probe = min(nlist, max(1, int(probe)))
164
+ cluster_scores = ann["centroids"] @ query_vector
165
+ clusters = np.argsort(-cluster_scores, kind="stable")[:probe]
166
+ parts = [ann["buckets"][int(cluster)] for cluster in clusters
167
+ if len(ann["buckets"][int(cluster)])]
168
+ if not parts:
169
+ raise _AnnFallback("empty ANN buckets")
170
+ positions = np.concatenate(parts)
171
+ if len(positions) < min(limit, len(matrix)):
172
+ raise _AnnFallback("too few ANN candidates")
173
+ scores = matrix[positions] @ query_vector
174
+ order = np.argsort(-scores, kind="stable")[:limit]
175
+ return positions[order], scores[order]
176
+
177
+
178
+ def rerank_hits(query: str, hits: list[dict], k: int = 5) -> list[dict]:
179
+ """Lexically rerank FTS candidates by coverage, title, phrase and bm25.
180
+
181
+ This is a deterministic reranker, not an embedding model. It is intentionally
182
+ cheap: local search should stay fast and work without a second model.
183
+ """
184
+ terms = _query_terms(query)
185
+ if not terms or not hits:
186
+ return hits[:k]
187
+ phrase = " ".join(str(query or "").lower().split())
188
+ query_vector = embed_text(query)
189
+ raw = []
190
+ for hit in hits:
191
+ value = hit.get("score")
192
+ try:
193
+ number = float(value) if value is not None else None
194
+ except (TypeError, ValueError):
195
+ number = None
196
+ raw.append(number if number is not None and math.isfinite(number)
197
+ else None)
198
+ finite = [value for value in raw if value is not None]
199
+ # bm25 returns lower-is-better; normalize it into 0..1. Embedding-only
200
+ # candidates have no bm25 and get 0 for this component.
201
+ high, low = (max(finite), min(finite)) if finite else (0.0, 0.0)
202
+ span = high - low
203
+ scored = []
204
+ for index, hit in enumerate(hits):
205
+ chunk = str(hit.get("chunk", "")).lower()
206
+ title = str(hit.get("title", "")).lower()
207
+ section = str(hit.get("section", "")).strip().lower()
208
+ searchable = f"{title} {section} {chunk}"
209
+ words = _query_terms(searchable)
210
+ coverage = len(terms & words) / len(terms)
211
+ title_hits = len(terms & _query_terms(f"{title} {section}")) / len(terms)
212
+ section_hits = (len(terms & _query_terms(section)) / len(terms)
213
+ if section else 0.0)
214
+ exact = 1.0 if len(terms) >= 2 and phrase in searchable else 0.0
215
+ if raw[index] is not None and span:
216
+ bm25 = 1.0 - ((raw[index] - low) / span)
217
+ elif raw[index] is not None:
218
+ bm25 = 0.5
219
+ else:
220
+ bm25 = 0.0
221
+ embedding = hit.get("embedding_score")
222
+ if embedding is None:
223
+ body_vector = embed_text(searchable)
224
+ embedding = sum(x * y for x, y in zip(query_vector, body_vector))
225
+ score = (0.40 * coverage + 0.10 * title_hits
226
+ + 0.05 * section_hits + 0.10 * exact
227
+ + 0.15 * bm25 + 0.25 * embedding)
228
+ item = dict(hit)
229
+ item["rerank_score"] = score
230
+ item["embedding_score"] = embedding
231
+ scored.append((score, index, item))
232
+ scored.sort(key=lambda row: (-row[0], row[1]))
233
+ return [item for _, _, item in scored[:k]]
234
+
235
+
236
+ def best_snippet(query: str, chunk: str, max_chars: int = 500) -> str:
237
+ """Select the one or two sentences that best match the query."""
238
+ text = str(chunk or "").strip()
239
+ if not text:
240
+ return ""
241
+ parts = [part.strip() for part in
242
+ re.split(r"(?<=[.!?])\s+|\n+", text) if part.strip()]
243
+ if len(parts) <= 1:
244
+ return text[:max_chars]
245
+ words = {word for word in re.findall(r"[a-z0-9]+", str(query).lower())
246
+ if len(word) > 2 and word not in STOPWORDS}
247
+ scores = [
248
+ sum(1 for word in re.findall(r"[a-z0-9]+", part.lower())
249
+ if word in words)
250
+ for part in parts
251
+ ]
252
+ chosen = sorted(sorted(range(len(parts)),
253
+ key=lambda i: (-scores[i], i))[:2])
254
+ snippet = " ".join(parts[i] for i in chosen)
255
+ if len(snippet) > max_chars:
256
+ snippet = parts[min(range(len(parts)),
257
+ key=lambda i: (-scores[i], i))]
258
+ return snippet[:max_chars]
259
+
260
+
261
+ def format_citation(hit: dict) -> str:
262
+ """Format a hit as ``source (p. N) [section]`` for chat/UI display."""
263
+ location = str(hit.get("source_url") or hit.get("source") or "")
264
+ page = hit.get("page")
265
+ if page not in (None, ""):
266
+ location += f" (p. {page})"
267
+ section = str(hit.get("section") or "").strip()
268
+ if section:
269
+ location += f" [{section}]"
270
+ return location
271
+
272
+
273
+ def add_citation_source(sources: dict, hit: dict) -> dict:
274
+ """Merge a page/section hit into a source-keyed citation map."""
275
+ key = str(hit.get("source", ""))
276
+ source = sources.get(key)
277
+ if source is None:
278
+ source = {"source": hit.get("source", ""),
279
+ "title": hit.get("title", "")}
280
+ for field in ("kind", "source_url", "teacher", "model", "verified",
281
+ "reviewer", "verified_at"):
282
+ if field in hit:
283
+ source[field] = hit[field]
284
+ sources[key] = source
285
+ page = hit.get("page")
286
+ if page not in (None, ""):
287
+ pages = source.setdefault("pages", [])
288
+ if page not in pages:
289
+ pages.append(page)
290
+ section = str(hit.get("section") or "").strip()
291
+ if section:
292
+ sections = source.setdefault("sections", [])
293
+ if section not in sections:
294
+ sections.append(section)
295
+ return source
296
+
297
+
298
+ def _chunks(text: str) -> list[str]:
299
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
300
+ paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
301
+ out = []
302
+ current = ""
303
+ for para in paragraphs:
304
+ if len(para) > MAX_CHARS:
305
+ if current:
306
+ out.append(current)
307
+ current = ""
308
+ out.extend(_split_long(para))
309
+ continue
310
+ if len(current) + len(para) + 2 <= MAX_CHARS:
311
+ current = (current + "\n\n" + para).strip()
312
+ else:
313
+ if current:
314
+ out.append(current)
315
+ current = para
316
+ if current:
317
+ out.append(current)
318
+ return out
319
+
320
+
321
+ class DocMemory:
322
+ def __init__(self, path: str | Path = "data/docs.db"):
323
+ self.path = Path(path)
324
+ self.path.parent.mkdir(parents=True, exist_ok=True)
325
+ self._lock = threading.RLock()
326
+ self.reranker = None
327
+ self._vector_index = None
328
+ self.vector_index_mode = "auto" # auto | ann | exact
329
+ self.ann_min_chunks = ANN_MIN_CHUNKS
330
+ self.ann_probe = None
331
+ self._vector_fallback_count = 0
332
+ self.con = sqlite3.connect(str(self.path), check_same_thread=False)
333
+ self.con.execute(
334
+ "CREATE VIRTUAL TABLE IF NOT EXISTS docs USING fts5("
335
+ "chunk, source UNINDEXED, title UNINDEXED, sha UNINDEXED, "
336
+ "ts UNINDEXED)")
337
+ self.con.execute(
338
+ "CREATE TABLE IF NOT EXISTS doc_meta("
339
+ "source TEXT PRIMARY KEY, kind TEXT NOT NULL, "
340
+ "source_url TEXT NOT NULL, teacher TEXT NOT NULL, "
341
+ "model TEXT NOT NULL, verified INTEGER NOT NULL, "
342
+ "created TEXT NOT NULL, reviewer TEXT NOT NULL DEFAULT '', "
343
+ "verified_at TEXT NOT NULL DEFAULT '')")
344
+ self.con.execute(
345
+ "CREATE TABLE IF NOT EXISTS doc_vectors("
346
+ "doc_rowid INTEGER NOT NULL, model TEXT NOT NULL, "
347
+ "dim INTEGER NOT NULL, vec BLOB NOT NULL, "
348
+ "PRIMARY KEY(doc_rowid, model))")
349
+ self.con.execute(
350
+ "CREATE TABLE IF NOT EXISTS doc_chunk_meta("
351
+ "doc_rowid INTEGER PRIMARY KEY, page INTEGER, "
352
+ "section TEXT NOT NULL DEFAULT '', "
353
+ "char_start INTEGER, char_end INTEGER)")
354
+ columns = {row[1] for row in self.con.execute(
355
+ "PRAGMA table_info(doc_meta)")}
356
+ for name, ddl in (
357
+ ("reviewer", "TEXT NOT NULL DEFAULT ''"),
358
+ ("verified_at", "TEXT NOT NULL DEFAULT ''")):
359
+ if name not in columns:
360
+ self.con.execute(
361
+ f"ALTER TABLE doc_meta ADD COLUMN {name} {ddl}")
362
+ self.con.commit()
363
+
364
+ def add_text(self, source: str, text: str, title: str = "",
365
+ replace: bool = True, meta: dict | None = None) -> int:
366
+ return self.add_blocks(
367
+ source, [{"text": text, "page": None, "section": ""}],
368
+ title=title, replace=replace, meta=meta)
369
+
370
+ def add_blocks(self, source: str, blocks: list[dict], title: str = "",
371
+ replace: bool = True, meta: dict | None = None) -> int:
372
+ """Index text blocks while keeping page/section provenance.
373
+
374
+ A block is a piece of text with an optional 1-based ``page`` and a
375
+ ``section`` label. Each block is split with the normal chunker; the
376
+ resulting chunks inherit the block's provenance and a character range
377
+ inside that block.
378
+ """
379
+ source = str(source)
380
+ title = str(title or Path(source).stem)
381
+ meta = meta or {}
382
+ kind = str(meta.get("kind", "document"))
383
+ source_url = str(meta.get("source_url", ""))
384
+ teacher = str(meta.get("teacher", ""))
385
+ model = str(meta.get("model", ""))
386
+ verified = 1 if meta.get("verified", True) else 0
387
+ created = str(meta.get("created", int(time.time())))
388
+ reviewer = str(meta.get("reviewer", ""))
389
+ verified_at = str(meta.get("verified_at", ""))
390
+ block_texts = [str(block.get("text", "")) for block in blocks]
391
+ sha = hashlib.sha256(
392
+ "\n\n".join(block_texts).encode("utf-8")).hexdigest()
393
+ with self._lock:
394
+ if replace:
395
+ self.con.execute("DELETE FROM docs WHERE source = ?",
396
+ (source,))
397
+ added = 0
398
+ for block in blocks:
399
+ text = str(block.get("text", "")).replace(
400
+ "\r\n", "\n").replace("\r", "\n").strip()
401
+ if not text:
402
+ continue
403
+ page = block.get("page")
404
+ try:
405
+ page = int(page) if page not in (None, "") else None
406
+ except (TypeError, ValueError):
407
+ page = None
408
+ section = str(block.get("section", "") or "").strip()
409
+ cursor = 0
410
+ for chunk in _chunks(text):
411
+ position = text.find(chunk, cursor)
412
+ if position < 0:
413
+ position = cursor
414
+ char_start = position
415
+ char_end = position + len(chunk)
416
+ inserted = self.con.execute(
417
+ "INSERT INTO docs(chunk, source, title, sha, ts) "
418
+ "VALUES (?, ?, ?, ?, ?)",
419
+ (chunk, source, title, sha, int(time.time())))
420
+ rowid = inserted.lastrowid
421
+ if not rowid:
422
+ rowid = self.con.execute(
423
+ "SELECT last_insert_rowid()").fetchone()[0]
424
+ self.con.execute(
425
+ "INSERT OR REPLACE INTO doc_chunk_meta("
426
+ "doc_rowid, page, section, char_start, char_end) "
427
+ "VALUES (?, ?, ?, ?, ?)",
428
+ (rowid, page, section, char_start, char_end))
429
+ cursor = char_end
430
+ added += 1
431
+ self.con.execute(
432
+ "INSERT OR REPLACE INTO doc_meta("
433
+ "source, kind, source_url, teacher, model, verified, created, "
434
+ "reviewer, verified_at) "
435
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
436
+ (source, kind, source_url, teacher, model, verified, created,
437
+ reviewer, verified_at))
438
+ if replace:
439
+ self._drop_stale_vectors()
440
+ self._drop_stale_chunk_meta()
441
+ self._vector_index = None
442
+ self.con.commit()
443
+ return added
444
+
445
+ def add_file(self, path: str | Path, title: str = "",
446
+ meta: dict | None = None) -> int:
447
+ from atomzero.doc_ingest import extract_file
448
+
449
+ p = Path(path)
450
+ extracted = extract_file(p, title=title)
451
+ return self.add_blocks(
452
+ str(p), extracted["blocks"], title=extracted["title"],
453
+ meta=meta)
454
+
455
+ def _drop_stale_vectors(self) -> None:
456
+ self.con.execute(
457
+ "DELETE FROM doc_vectors WHERE doc_rowid NOT IN "
458
+ "(SELECT rowid FROM docs)")
459
+
460
+ def _drop_stale_chunk_meta(self) -> None:
461
+ self.con.execute(
462
+ "DELETE FROM doc_chunk_meta WHERE doc_rowid NOT IN "
463
+ "(SELECT rowid FROM docs)")
464
+
465
+ @staticmethod
466
+ def _vector_model_key(active) -> str:
467
+ if active is None or not hasattr(active, "embed"):
468
+ return HASHED_CACHE_KEY
469
+ key = getattr(active, "cache_key", "")
470
+ if key:
471
+ return str(key)
472
+ path = getattr(active, "model_path", "")
473
+ if path:
474
+ return f"neural:{Path(path).resolve()}"
475
+ return "neural:custom"
476
+
477
+ @staticmethod
478
+ def _embed_texts(active, texts: list[str]) -> list[list[float]]:
479
+ if active is not None and hasattr(active, "embed"):
480
+ vectors = active.embed(texts)
481
+ return [[float(value) for value in row] for row in vectors.tolist()]
482
+ return [embed_text(text) for text in texts]
483
+
484
+ def _vectors_for_rows(self, active, rows) -> dict[int, list[float]]:
485
+ """Load cached chunk vectors, computing and storing missing ones."""
486
+ if not rows:
487
+ return {}
488
+ key = self._vector_model_key(active)
489
+ wanted = {rowid: text for rowid, text in rows}
490
+ vectors: dict[int, list[float]] = {}
491
+ ids = list(wanted)
492
+ for start in range(0, len(ids), 400):
493
+ batch = ids[start:start + 400]
494
+ placeholders = ",".join("?" for _ in batch)
495
+ found = self.con.execute(
496
+ "SELECT doc_rowid, vec FROM doc_vectors "
497
+ f"WHERE model = ? AND doc_rowid IN ({placeholders})",
498
+ (key, *batch)).fetchall()
499
+ for rowid, blob in found:
500
+ vectors[rowid] = _unpack_vector(blob)
501
+ missing = [rowid for rowid in ids if rowid not in vectors]
502
+ if missing:
503
+ computed = self._embed_texts(
504
+ active, [wanted[rowid] for rowid in missing])
505
+ self.con.executemany(
506
+ "INSERT OR REPLACE INTO doc_vectors"
507
+ "(doc_rowid, model, dim, vec) VALUES (?, ?, ?, ?)",
508
+ [(rowid, key, len(vector), _pack_vector(vector))
509
+ for rowid, vector in zip(missing, computed)])
510
+ self.con.commit()
511
+ for rowid, vector in zip(missing, computed):
512
+ vectors[rowid] = vector
513
+ return vectors
514
+
515
+ def prewarm_vectors(self, active=None) -> dict:
516
+ """Build the in-memory matrix over all current chunk vectors.
517
+
518
+ With no explicit reranker, use the attached one (neural when
519
+ configured, hashed otherwise).
520
+ """
521
+ if active is None:
522
+ active = self.reranker
523
+ started = time.perf_counter()
524
+ with self._lock:
525
+ rows = self.con.execute(
526
+ "SELECT rowid, chunk, source, title FROM docs "
527
+ "ORDER BY rowid").fetchall()
528
+ key = self._vector_model_key(active)
529
+ if not rows:
530
+ self._vector_index = {
531
+ "key": key, "rows": [], "rowids": [], "matrix": None,
532
+ "positions": {}, "build_s": 0.0}
533
+ return self.vector_index_stats()
534
+ texts = [(row[0], f"{row[3]} {row[1]}".strip()) for row in rows]
535
+ vectors = self._vectors_for_rows(active, texts)
536
+ kept_rows = []
537
+ kept_vectors = []
538
+ for row in rows:
539
+ vector = vectors.get(row[0])
540
+ if vector is None:
541
+ continue
542
+ if kept_vectors and len(vector) != len(kept_vectors[0]):
543
+ continue
544
+ kept_rows.append(row)
545
+ kept_vectors.append(vector)
546
+ matrix = None
547
+ if np is not None and kept_vectors:
548
+ matrix = np.asarray(kept_vectors, dtype=np.float32)
549
+ matrix = _normalized_rows(matrix)
550
+ mode = self._resolved_vector_mode(len(kept_rows))
551
+ ann = None
552
+ ann_build_s = 0.0
553
+ if mode == "ann" and matrix is not None:
554
+ ann_started = time.perf_counter()
555
+ try:
556
+ ann = build_ivf(matrix)
557
+ except (MemoryError, RuntimeError, ValueError):
558
+ ann = None
559
+ ann_build_s = time.perf_counter() - ann_started
560
+ if ann is None:
561
+ mode = "exact"
562
+ rowids = [row[0] for row in kept_rows]
563
+ self._vector_index = {
564
+ "key": key,
565
+ "rows": kept_rows,
566
+ "rowids": rowids,
567
+ "matrix": matrix,
568
+ "positions": {rowid: index
569
+ for index, rowid in enumerate(rowids)},
570
+ "mode": mode,
571
+ "ann": ann,
572
+ "ann_build_s": ann_build_s,
573
+ "build_s": time.perf_counter() - started,
574
+ }
575
+ self._vector_fallback_count = 0
576
+ return self.vector_index_stats()
577
+
578
+ def _resolved_vector_mode(self, chunks: int) -> str:
579
+ if self.vector_index_mode == "exact" or np is None:
580
+ return "exact"
581
+ if self.vector_index_mode == "ann":
582
+ return "ann" if chunks >= 2 else "exact"
583
+ return ("ann" if chunks >= max(2, int(self.ann_min_chunks))
584
+ else "exact")
585
+
586
+ def vector_index_stats(self) -> dict:
587
+ index = self._vector_index
588
+ backend = "numpy" if np is not None else "scan"
589
+ if not index:
590
+ return {"built": False, "chunks": 0, "dim": 0,
591
+ "build_s": 0.0, "backend": backend}
592
+ matrix = index.get("matrix")
593
+ ann = index.get("ann")
594
+ nlist = int(ann["nlist"]) if ann else 0
595
+ return {
596
+ "built": matrix is not None,
597
+ "chunks": len(index.get("rows", [])),
598
+ "dim": int(matrix.shape[1]) if matrix is not None else 0,
599
+ "mode": index.get("mode", "exact"),
600
+ "ann": bool(ann),
601
+ "nlist": nlist,
602
+ "nprobe": (int(self.ann_probe) if self.ann_probe
603
+ else (max(1, int(math.ceil(math.sqrt(nlist))))
604
+ if nlist else 0)),
605
+ "exact_fallback": True,
606
+ "fallbacks": int(self._vector_fallback_count),
607
+ "build_s": round(float(index.get("build_s", 0.0)), 3),
608
+ "ann_build_s": round(float(index.get("ann_build_s", 0.0)), 3),
609
+ "backend": backend,
610
+ }
611
+
612
+ @staticmethod
613
+ def _candidate_filters(query: str, neural: bool):
614
+ threshold = HYBRID_MIN_NEURAL if neural else HYBRID_MIN_HASHED
615
+ grams = set(_char_ngrams(query)) if not neural else None
616
+ return threshold, grams
617
+
618
+ @staticmethod
619
+ def _has_hashed_overlap(query_grams, row) -> bool:
620
+ if query_grams is None:
621
+ return True
622
+ # Hashed vectors are sparse and can collide on short queries. Require
623
+ # real character overlap before a hash-only candidate may bypass FTS.
624
+ text = f"{row[3]} {row[1]}".lower()
625
+ return len(query_grams & set(_char_ngrams(text))) >= 2
626
+
627
+ @staticmethod
628
+ def _exact_candidates(index, query_vector, limit: int):
629
+ scores = index["matrix"] @ query_vector
630
+ order = np.argsort(-scores, kind="stable")[:limit]
631
+ return order, scores[order]
632
+
633
+ def _search_positions(self, index, query_vector, limit: int,
634
+ required_rowids, threshold: float):
635
+ matrix = index["matrix"]
636
+ ann = index.get("ann")
637
+ positions = None
638
+ scores = None
639
+ if ann is not None:
640
+ try:
641
+ positions, scores = ann_topk(
642
+ matrix, ann, query_vector, limit, self.ann_probe)
643
+ if not len(positions) or float(scores[0]) < threshold:
644
+ raise _AnnFallback("low ANN confidence")
645
+ except Exception:
646
+ positions = None
647
+ scores = None
648
+ if positions is None:
649
+ if ann is not None:
650
+ self._vector_fallback_count += 1
651
+ positions, scores = self._exact_candidates(
652
+ index, query_vector, limit)
653
+ by_rowid = {
654
+ index["rowids"][int(position)]: float(score)
655
+ for position, score in zip(positions, scores)
656
+ }
657
+ required = [index["positions"][rowid]
658
+ for rowid in required_rowids or ()
659
+ if rowid in index["positions"]]
660
+ if required:
661
+ extra = matrix[required] @ query_vector
662
+ for position, score in zip(required, extra):
663
+ by_rowid[index["rowids"][position]] = float(score)
664
+ return ([int(position) for position in positions],
665
+ [float(score) for score in scores], by_rowid)
666
+
667
+ def _embedding_candidates(self, query: str, active, limit: int,
668
+ required_rowids=None):
669
+ """Score chunks with the prewarmed matrix; rebuild it when invalid."""
670
+ key = self._vector_model_key(active)
671
+ index = self._vector_index
672
+ if index is None or index.get("key") != key:
673
+ self.prewarm_vectors(active)
674
+ index = self._vector_index
675
+ matrix = index.get("matrix") if index else None
676
+ if matrix is None:
677
+ return self._embedding_candidates_scan(
678
+ query, active, limit, required_rowids)
679
+ if not index["rows"]:
680
+ return [], {}
681
+ query_vector = np.asarray(
682
+ self._embed_texts(active, [query])[0], dtype=np.float32)
683
+ if matrix.shape[1] != query_vector.shape[0]:
684
+ return self._embedding_candidates_scan(
685
+ query, active, limit, required_rowids)
686
+ neural = key.startswith("neural:")
687
+ threshold, query_grams = self._candidate_filters(query, neural)
688
+ try:
689
+ top, top_scores, by_rowid = self._search_positions(
690
+ index, query_vector, limit, required_rowids, threshold)
691
+ except (MemoryError, RuntimeError, ValueError, IndexError):
692
+ return self._embedding_candidates_scan(
693
+ query, active, limit, required_rowids)
694
+ hits = []
695
+ for position, similarity in zip(top, top_scores):
696
+ if similarity < threshold:
697
+ break
698
+ row = index["rows"][position]
699
+ if not self._has_hashed_overlap(query_grams, row):
700
+ continue
701
+ hit = {
702
+ "chunk": row[1], "source": row[2], "title": row[3],
703
+ "score": None, "_rowid": row[0],
704
+ "embedding_score": similarity,
705
+ }
706
+ if neural:
707
+ hit["neural_score"] = similarity
708
+ hits.append(hit)
709
+ return hits, by_rowid
710
+
711
+ def _embedding_candidates_scan(self, query: str, active, limit: int,
712
+ required_rowids=None):
713
+ """Legacy linear scan; used only when numpy is unavailable."""
714
+ rows = self.con.execute(
715
+ "SELECT rowid, chunk, source, title FROM docs "
716
+ "ORDER BY rowid").fetchall()
717
+ if not rows:
718
+ return [], {}
719
+ texts = [(row[0], f"{row[3]} {row[1]}".strip()) for row in rows]
720
+ vectors = self._vectors_for_rows(active, texts)
721
+ query_vector = self._embed_texts(active, [query])[0]
722
+ neural = self._vector_model_key(active).startswith("neural:")
723
+ scored = []
724
+ for row in rows:
725
+ vector = vectors.get(row[0])
726
+ if vector is None or len(vector) != len(query_vector):
727
+ continue
728
+ similarity = sum(a * b for a, b in
729
+ zip(query_vector, vector))
730
+ scored.append((similarity, row))
731
+ scored.sort(key=lambda item: (-item[0], item[1][0]))
732
+ by_rowid = {row[0]: similarity for similarity, row in scored}
733
+ threshold, query_grams = self._candidate_filters(query, neural)
734
+ hits = []
735
+ for similarity, row in scored[:limit]:
736
+ if similarity < threshold:
737
+ break
738
+ if not self._has_hashed_overlap(query_grams, row):
739
+ continue
740
+ hit = {
741
+ "chunk": row[1], "source": row[2], "title": row[3],
742
+ "score": None, "_rowid": row[0],
743
+ "embedding_score": similarity,
744
+ }
745
+ if neural:
746
+ hit["neural_score"] = similarity
747
+ hits.append(hit)
748
+ return hits, by_rowid
749
+
750
+ def _meta_map(self, sources) -> dict[str, dict]:
751
+ sources = list(sources)
752
+ if not sources:
753
+ return {}
754
+ placeholders = ",".join("?" for _ in sources)
755
+ rows = self.con.execute(
756
+ "SELECT source, kind, source_url, teacher, model, verified, "
757
+ f"created, reviewer, verified_at FROM doc_meta "
758
+ f"WHERE source IN ({placeholders})",
759
+ tuple(sources)).fetchall()
760
+ return {row[0]: {
761
+ "source": row[0],
762
+ "kind": row[1],
763
+ "source_url": row[2],
764
+ "teacher": row[3],
765
+ "model": row[4],
766
+ "verified": bool(row[5]),
767
+ "created": row[6],
768
+ "reviewer": row[7],
769
+ "verified_at": row[8],
770
+ } for row in rows}
771
+
772
+ def _chunk_meta_map(self, rowids) -> dict[int, dict]:
773
+ rowids = [int(rowid) for rowid in rowids
774
+ if rowid is not None]
775
+ if not rowids:
776
+ return {}
777
+ placeholders = ",".join("?" for _ in rowids)
778
+ rows = self.con.execute(
779
+ "SELECT doc_rowid, page, section, char_start, char_end "
780
+ f"FROM doc_chunk_meta WHERE doc_rowid IN ({placeholders})",
781
+ tuple(rowids)).fetchall()
782
+ return {row[0]: {
783
+ "page": row[1],
784
+ "section": row[2] or "",
785
+ "char_start": row[3],
786
+ "char_end": row[4],
787
+ } for row in rows}
788
+
789
+ def search(self, query: str, k: int = 5,
790
+ rerank: bool = True,
791
+ reranker=None) -> list[dict]:
792
+ words = re.findall(r"[\wçğıöşüÇĞİÖŞÜ]+", query)
793
+ words = [w for w in words if len(w) > 1]
794
+ meaningful = [w for w in words if w.lower() not in STOPWORDS]
795
+ if meaningful:
796
+ words = meaningful
797
+ if not words:
798
+ return []
799
+ match = " OR ".join(f'"{w}"' for w in words[:12])
800
+ candidates = max(k * 4, 12) if rerank else k
801
+ active = reranker if reranker is not None else self.reranker
802
+ neural = self._vector_model_key(active).startswith("neural:")
803
+ with self._lock:
804
+ rows = self.con.execute(
805
+ "SELECT rowid, chunk, source, title, bm25(docs) AS score "
806
+ "FROM docs WHERE docs MATCH ? ORDER BY score LIMIT ?",
807
+ (match, candidates)).fetchall()
808
+ embedding_hits, embedding_scores = ([], {})
809
+ # Hashed n-grams are a fallback reranker, not a semantic retriever:
810
+ # do not answer from hash-only candidates when FTS found nothing.
811
+ # The neural reranker has a calibrated threshold and may do so.
812
+ if rerank and (rows or neural):
813
+ embedding_hits, embedding_scores = \
814
+ self._embedding_candidates(
815
+ query, active, candidates,
816
+ required_rowids={row[0] for row in rows})
817
+ sources = {row[2] for row in rows} | {
818
+ hit["source"] for hit in embedding_hits}
819
+ metas = self._meta_map(sources)
820
+ chunk_metas = self._chunk_meta_map(
821
+ {row[0] for row in rows}
822
+ | {hit.get("_rowid") for hit in embedding_hits})
823
+ default = {"kind": "document", "source_url": "", "teacher": "",
824
+ "model": "", "verified": True, "created": "",
825
+ "reviewer": "", "verified_at": "", "page": None,
826
+ "section": "", "char_start": None, "char_end": None}
827
+ hits = [{
828
+ "chunk": row[1],
829
+ "source": row[2],
830
+ "title": row[3],
831
+ "score": float(row[4]),
832
+ "_rowid": row[0],
833
+ **default,
834
+ **metas.get(row[2], {}),
835
+ **chunk_metas.get(row[0], {}),
836
+ } for row in rows]
837
+ for hit in embedding_hits:
838
+ hit.update(chunk_metas.get(hit.get("_rowid"), {}))
839
+ if not rerank:
840
+ return hits
841
+ # Hybrid candidate pool: FTS rows plus the best embedding rows. A
842
+ # paraphrase with no lexical overlap with any document can now enter
843
+ # the reranker, which was previously capped by FTS recall.
844
+ merged = []
845
+ seen = set()
846
+ for hit in hits + embedding_hits:
847
+ key = (hit["source"], hit["chunk"])
848
+ if key in seen:
849
+ continue
850
+ seen.add(key)
851
+ merged.append(hit)
852
+ finite = [hit["score"] for hit in hits
853
+ if hit.get("score") is not None]
854
+ fallback = max(finite) if finite else None
855
+ for hit in merged:
856
+ similarity = embedding_scores.get(hit.get("_rowid"))
857
+ if similarity is not None:
858
+ hit["embedding_score"] = similarity
859
+ if neural:
860
+ hit["neural_score"] = similarity
861
+ if hit.get("score") is None:
862
+ hit["score"] = fallback
863
+ if active is not None:
864
+ ranked = active.rerank(query, merged, k)
865
+ else:
866
+ ranked = rerank_hits(query, merged, k)
867
+ for hit in ranked:
868
+ hit.pop("_rowid", None)
869
+ return ranked
870
+
871
+ def sources(self) -> list[dict]:
872
+ with self._lock:
873
+ rows = self.con.execute(
874
+ "SELECT source, title, COUNT(*), MAX(sha) FROM docs "
875
+ "GROUP BY source, title ORDER BY source").fetchall()
876
+ metas = self._meta_map([row[0] for row in rows])
877
+ page_rows = self.con.execute(
878
+ "SELECT docs.source, MIN(m.page), MAX(m.page) "
879
+ "FROM doc_chunk_meta m JOIN docs ON docs.rowid = m.doc_rowid "
880
+ "GROUP BY docs.source").fetchall()
881
+ pages = {row[0]: (row[1], row[2]) for row in page_rows}
882
+ result = []
883
+ for row in rows:
884
+ item = {
885
+ "source": row[0],
886
+ "title": row[1],
887
+ "chunks": row[2],
888
+ "content_sha256": row[3],
889
+ **metas.get(row[0], {"source": row[0], "kind": "document",
890
+ "source_url": "", "teacher": "",
891
+ "model": "", "verified": True,
892
+ "created": "", "reviewer": "",
893
+ "verified_at": ""}),
894
+ }
895
+ page_range = pages.get(row[0])
896
+ if page_range and page_range[0] is not None:
897
+ item["page_start"] = page_range[0]
898
+ item["page_end"] = page_range[1]
899
+ result.append(item)
900
+ return result
901
+
902
+ def source_evidence(self, source: str, page: int | None = None,
903
+ section: str = "", limit: int = 20) -> dict:
904
+ """Return the stored chunks for a source/page/section locator."""
905
+ source = str(source or "").strip()
906
+ if not source:
907
+ raise ValueError("source is required")
908
+ if len(source) > 512:
909
+ raise ValueError("source is too long (max 512 chars)")
910
+ if any(ord(ch) < 32 for ch in source):
911
+ raise ValueError("source contains control characters")
912
+ try:
913
+ limit = int(limit)
914
+ except (TypeError, ValueError) as exc:
915
+ raise ValueError("limit must be an integer") from exc
916
+ limit = max(1, min(limit, 50))
917
+ clauses = ["d.source = ?"]
918
+ params: list = [source]
919
+ if page is not None:
920
+ try:
921
+ page = int(page)
922
+ except (TypeError, ValueError) as exc:
923
+ raise ValueError("page must be an integer") from exc
924
+ if page < 1:
925
+ raise ValueError("page must be >= 1")
926
+ clauses.append("m.page = ?")
927
+ params.append(page)
928
+ section = str(section or "").strip()
929
+ if section:
930
+ if len(section) > 200:
931
+ raise ValueError("section is too long (max 200 chars)")
932
+ clauses.append("lower(m.section) = lower(?)")
933
+ params.append(section)
934
+ with self._lock:
935
+ rows = self.con.execute(
936
+ "SELECT d.chunk, d.source, d.title, m.page, m.section, "
937
+ "m.char_start, m.char_end FROM docs d "
938
+ "LEFT JOIN doc_chunk_meta m ON m.doc_rowid = d.rowid "
939
+ f"WHERE {' AND '.join(clauses)} "
940
+ "ORDER BY COALESCE(m.page, 0), d.rowid LIMIT ?",
941
+ (*params, limit)).fetchall()
942
+ if not rows:
943
+ raise SourceNotFound(
944
+ f"no evidence for source/page/section: {source}")
945
+ data = [{
946
+ "chunk": row[0],
947
+ "source": row[1],
948
+ "title": row[2],
949
+ "page": row[3],
950
+ "section": row[4] or "",
951
+ "char_start": row[5],
952
+ "char_end": row[6],
953
+ } for row in rows]
954
+ return {
955
+ "source": source,
956
+ "page": page,
957
+ "section": section,
958
+ "returned": len(data),
959
+ "data": data,
960
+ }
961
+
962
+ def verify_source(self, source: str, reviewer: str) -> dict:
963
+ """Mark a teacher note verified with reviewer and timestamp."""
964
+ source = str(source or "").strip()
965
+ reviewer = str(reviewer or "").strip()
966
+ if not source:
967
+ raise ValueError("source is required")
968
+ if not reviewer:
969
+ raise ValueError("reviewer is required")
970
+ if len(reviewer) > 100:
971
+ raise ValueError("reviewer is too long (max 100 chars)")
972
+ if any(ord(ch) < 32 for ch in reviewer):
973
+ raise ValueError("reviewer contains control characters")
974
+ with self._lock:
975
+ row = self.con.execute(
976
+ "SELECT kind FROM doc_meta WHERE source = ?",
977
+ (source,)).fetchone()
978
+ if row is None:
979
+ raise SourceNotFound(f"source not found: {source}")
980
+ if row[0] != "teacher":
981
+ raise ValueError("only teacher notes can be verified")
982
+ verified_at = time.strftime("%Y-%m-%dT%H:%M:%S")
983
+ self.con.execute(
984
+ "UPDATE doc_meta SET verified = 1, reviewer = ?, "
985
+ "verified_at = ? WHERE source = ?",
986
+ (reviewer, verified_at, source))
987
+ self.con.commit()
988
+ meta = self._meta_map([source]).get(source, {})
989
+ return {"source": source, **meta}
990
+
991
+ def forget(self, source: str) -> dict:
992
+ """Delete one source and its metadata, then compact the database."""
993
+ source = str(source or "").strip()
994
+ if not source:
995
+ raise ValueError("source is required")
996
+ variants = [source]
997
+ if "://" not in source:
998
+ # Windows stores local paths with backslashes; accept either form
999
+ # on every platform (Path() treats a backslash as a normal char on
1000
+ # POSIX, so do the replacement explicitly too).
1001
+ for variant in (source.replace("/", "\\"),
1002
+ source.replace("\\", "/"),
1003
+ Path(source).as_posix(), str(Path(source))):
1004
+ if variant not in variants:
1005
+ variants.append(variant)
1006
+ placeholders = ",".join("?" for _ in variants)
1007
+ with self._lock:
1008
+ chunks = self.con.execute(
1009
+ f"SELECT COUNT(*) FROM docs WHERE source IN ({placeholders})",
1010
+ tuple(variants)).fetchone()[0]
1011
+ meta = self.con.execute(
1012
+ f"SELECT COUNT(*) FROM doc_meta WHERE source IN "
1013
+ f"({placeholders})", tuple(variants)).fetchone()[0]
1014
+ if chunks == 0 and meta == 0:
1015
+ raise SourceNotFound(f"source not found: {source}")
1016
+ self.con.execute(
1017
+ f"DELETE FROM docs WHERE source IN ({placeholders})",
1018
+ tuple(variants))
1019
+ self.con.execute(
1020
+ f"DELETE FROM doc_meta WHERE source IN ({placeholders})",
1021
+ tuple(variants))
1022
+ self._drop_stale_vectors()
1023
+ self._drop_stale_chunk_meta()
1024
+ self._vector_index = None
1025
+ self.con.commit()
1026
+ vacuumed = False
1027
+ try:
1028
+ self.con.execute("VACUUM")
1029
+ vacuumed = True
1030
+ except sqlite3.OperationalError:
1031
+ pass
1032
+ return {"source": source, "chunks_deleted": int(chunks),
1033
+ "meta_deleted": int(meta), "vacuumed": vacuumed}
1034
+
1035
+ def forget_all(self) -> dict:
1036
+ """Delete every indexed source and compact the database."""
1037
+ with self._lock:
1038
+ chunks = self.con.execute(
1039
+ "SELECT COUNT(*) FROM docs").fetchone()[0]
1040
+ meta = self.con.execute(
1041
+ "SELECT COUNT(*) FROM doc_meta").fetchone()[0]
1042
+ self.con.execute("DELETE FROM docs")
1043
+ self.con.execute("DELETE FROM doc_meta")
1044
+ self._drop_stale_vectors()
1045
+ self._drop_stale_chunk_meta()
1046
+ self._vector_index = None
1047
+ self.con.commit()
1048
+ vacuumed = False
1049
+ try:
1050
+ self.con.execute("VACUUM")
1051
+ vacuumed = True
1052
+ except sqlite3.OperationalError:
1053
+ pass
1054
+ return {"chunks_deleted": int(chunks),
1055
+ "meta_deleted": int(meta), "vacuumed": vacuumed}
1056
+
1057
+ def stats(self) -> dict:
1058
+ with self._lock:
1059
+ count = self.con.execute(
1060
+ "SELECT COUNT(*) FROM docs").fetchone()[0]
1061
+ return {"path": str(self.path), "chunks": count,
1062
+ "sources": len(self.sources())}
1063
+
1064
+ def close(self):
1065
+ with self._lock:
1066
+ self.con.close()
1067
+
1068
+
1069
+ def main(argv=None):
1070
+ ap = argparse.ArgumentParser()
1071
+ ap.add_argument("--db", default=None)
1072
+ sub = ap.add_subparsers(dest="cmd", required=True)
1073
+ add = sub.add_parser("add")
1074
+ add.add_argument("path")
1075
+ add.add_argument("--title", default="")
1076
+ search = sub.add_parser("search")
1077
+ search.add_argument("query")
1078
+ search.add_argument("-k", type=int, default=5)
1079
+ list_db = sub.add_parser("list")
1080
+ stats_db = sub.add_parser("stats")
1081
+ forget = sub.add_parser("forget")
1082
+ forget.add_argument("source", nargs="?")
1083
+ forget.add_argument("--all", action="store_true")
1084
+ # Accept --db on either side of the subcommand; the docs use both forms.
1085
+ for parser in (add, search, list_db, stats_db, forget):
1086
+ parser.add_argument("--db", dest="db_after", default=None)
1087
+ args = ap.parse_args(argv)
1088
+ mem = DocMemory(args.db_after or args.db or "data/docs.db")
1089
+ if args.cmd == "add":
1090
+ print(json.dumps({"added": mem.add_file(args.path, args.title)}))
1091
+ elif args.cmd == "search":
1092
+ print(json.dumps(mem.search(args.query, args.k), indent=2,
1093
+ ensure_ascii=False))
1094
+ elif args.cmd == "list":
1095
+ print(json.dumps(mem.sources(), indent=2, ensure_ascii=False))
1096
+ elif args.cmd == "forget":
1097
+ try:
1098
+ result = (mem.forget_all() if args.all
1099
+ else mem.forget(args.source or ""))
1100
+ except (ValueError, SourceNotFound) as exc:
1101
+ print(json.dumps({"error": str(exc)}))
1102
+ mem.close()
1103
+ return 1
1104
+ print(json.dumps(result, ensure_ascii=False))
1105
+ else:
1106
+ print(json.dumps(mem.stats(), indent=2))
1107
+ mem.close()
1108
+ return 0
1109
+
1110
+
1111
+ if __name__ == "__main__":
1112
+ raise SystemExit(main())