mindslice 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. mindslice/__init__.py +58 -0
  2. mindslice/__main__.py +5 -0
  3. mindslice/baking/__init__.py +1 -0
  4. mindslice/baking/archive.py +66 -0
  5. mindslice/baking/bake.py +299 -0
  6. mindslice/baking/bake_batch.py +537 -0
  7. mindslice/baking/slice_transfer.py +265 -0
  8. mindslice/errors.py +104 -0
  9. mindslice/interfaces/__init__.py +1 -0
  10. mindslice/interfaces/bake_review.py +230 -0
  11. mindslice/interfaces/cli.py +392 -0
  12. mindslice/interfaces/cli_addressing.py +52 -0
  13. mindslice/interfaces/doctor.py +184 -0
  14. mindslice/interfaces/mcp_server.py +536 -0
  15. mindslice/interfaces/render.py +346 -0
  16. mindslice/interfaces/staging_review.py +92 -0
  17. mindslice/model/__init__.py +1 -0
  18. mindslice/model/addressing.py +251 -0
  19. mindslice/model/card.py +641 -0
  20. mindslice/model/lineage.py +228 -0
  21. mindslice/model/text.py +19 -0
  22. mindslice/model/tomlfmt.py +129 -0
  23. mindslice/operations/__init__.py +1 -0
  24. mindslice/operations/audit.py +195 -0
  25. mindslice/operations/boot.py +183 -0
  26. mindslice/operations/impact.py +98 -0
  27. mindslice/operations/recovery.py +113 -0
  28. mindslice/operations/staging.py +641 -0
  29. mindslice/operations/status.py +137 -0
  30. mindslice/operations/validation.py +250 -0
  31. mindslice/operations/writer.py +481 -0
  32. mindslice/retrieval/__init__.py +1 -0
  33. mindslice/retrieval/index.py +634 -0
  34. mindslice/retrieval/lexical.py +215 -0
  35. mindslice/retrieval/query.py +593 -0
  36. mindslice/storage/__init__.py +1 -0
  37. mindslice/storage/card_io.py +14 -0
  38. mindslice/storage/config.py +192 -0
  39. mindslice/storage/fsutil.py +141 -0
  40. mindslice/storage/history.py +182 -0
  41. mindslice/storage/locking.py +125 -0
  42. mindslice/storage/snapshots.py +262 -0
  43. mindslice/workspace.py +385 -0
  44. mindslice-0.1.0.dist-info/METADATA +98 -0
  45. mindslice-0.1.0.dist-info/RECORD +50 -0
  46. mindslice-0.1.0.dist-info/WHEEL +5 -0
  47. mindslice-0.1.0.dist-info/entry_points.txt +2 -0
  48. mindslice-0.1.0.dist-info/licenses/LICENSE +202 -0
  49. mindslice-0.1.0.dist-info/licenses/NOTICE +22 -0
  50. mindslice-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,634 @@
1
+ """Disposable SQLite acceleration cache (``.cache/index.sqlite``).
2
+
3
+ Everything except telemetry is reconstructible from the Markdown on disk:
4
+
5
+ * ``cards`` one row per card file (stat, sha256, parsed metadata, preview)
6
+ * ``cards_fts`` FTS5 table over ``title, aliases, tags, summary, body``
7
+ * ``links`` raw lineage / wikilink references plus their current resolution
8
+ * ``mounts`` snapshot of the mount table the index was built against
9
+ * ``telemetry`` best-effort ``read_count`` / ``last_read_at`` (survives rebuilds,
10
+ but is lost if the cache file is deleted)
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import math
17
+ import os
18
+ import sqlite3
19
+ import time
20
+ from dataclasses import dataclass, field
21
+ from pathlib import Path
22
+ from typing import Any, Iterable, Sequence
23
+
24
+ from ..model.addressing import MountCatalog, resolve_ref
25
+ from ..model.card import PARSER_VERSION, parse_card, preview_text, sha256_bytes
26
+ from ..storage.config import OVERVIEW_PATH, Mount, MountTable
27
+ from ..storage.fsutil import iter_markdown
28
+ from .lexical import QueryPlan, analyze_query, analyzer_signature, fold, index_text
29
+
30
+ SCHEMA_VERSION = "2" # table layout ("2": cards.load); text-analysis and parsing changes are tracked by index_signature()
31
+ BM25_WEIGHTS = (10.0, 5.0, 3.0, 2.0, 1.0) # title, aliases, tags, summary, body
32
+ TOKENIZER = "porter unicode61 remove_diacritics 2"
33
+ LINK_KINDS = ("extends", "supersedes", "wikilink")
34
+
35
+ _SCHEMA = f"""
36
+ CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);
37
+ CREATE TABLE IF NOT EXISTS mounts(
38
+ alias TEXT PRIMARY KEY, root TEXT NOT NULL, precedence INTEGER NOT NULL,
39
+ role TEXT NOT NULL, kind TEXT NOT NULL
40
+ );
41
+ CREATE TABLE IF NOT EXISTS cards(
42
+ id INTEGER PRIMARY KEY,
43
+ mount TEXT NOT NULL, path TEXT NOT NULL,
44
+ title TEXT NOT NULL, aliases TEXT NOT NULL, tags TEXT NOT NULL,
45
+ extends TEXT NOT NULL, supersedes TEXT NOT NULL,
46
+ summary TEXT NOT NULL, preview TEXT NOT NULL,
47
+ sha256 TEXT NOT NULL, mtime_ns INTEGER NOT NULL, size INTEGER NOT NULL,
48
+ lines INTEGER NOT NULL, is_overview INTEGER NOT NULL, has_frontmatter INTEGER NOT NULL,
49
+ load TEXT NOT NULL DEFAULT 'as-needed',
50
+ errors TEXT NOT NULL, warnings TEXT NOT NULL,
51
+ UNIQUE(mount, path)
52
+ );
53
+ CREATE VIRTUAL TABLE IF NOT EXISTS cards_fts USING fts5(
54
+ title, aliases, tags, summary, body, tokenize = '{TOKENIZER}'
55
+ );
56
+ CREATE TABLE IF NOT EXISTS card_aliases(mount TEXT NOT NULL, path TEXT NOT NULL, alias TEXT NOT NULL);
57
+ CREATE INDEX IF NOT EXISTS card_aliases_idx ON card_aliases(mount, alias);
58
+ CREATE TABLE IF NOT EXISTS card_tags(mount TEXT NOT NULL, path TEXT NOT NULL, tag TEXT NOT NULL);
59
+ DROP TABLE IF EXISTS card_words; -- left by analyzer "2" caches (did_you_mean, since removed)
60
+ CREATE INDEX IF NOT EXISTS card_tags_idx ON card_tags(mount, path);
61
+ CREATE TABLE IF NOT EXISTS links(
62
+ id INTEGER PRIMARY KEY,
63
+ src_mount TEXT NOT NULL, src_path TEXT NOT NULL, kind TEXT NOT NULL, ord INTEGER NOT NULL,
64
+ raw TEXT NOT NULL, dst_mount TEXT, dst_path TEXT, resolved INTEGER NOT NULL DEFAULT 0
65
+ );
66
+ CREATE INDEX IF NOT EXISTS links_src_idx ON links(src_mount, src_path);
67
+ CREATE INDEX IF NOT EXISTS links_dst_idx ON links(kind, dst_mount, dst_path);
68
+ CREATE TABLE IF NOT EXISTS telemetry(
69
+ mount TEXT NOT NULL, path TEXT NOT NULL, read_count INTEGER NOT NULL DEFAULT 0,
70
+ last_read_at REAL, PRIMARY KEY(mount, path)
71
+ );
72
+ """
73
+
74
+ _DERIVED_TABLES = ("cards", "cards_fts", "card_aliases", "card_tags", "links", "mounts")
75
+
76
+
77
+ @dataclass
78
+ class SyncStats:
79
+ added: int = 0
80
+ updated: int = 0
81
+ removed: int = 0
82
+ touched: int = 0 # stat changed but content identical
83
+ mounts_changed: bool = False
84
+ changed_cards: list[tuple[str, str]] = field(default_factory=list)
85
+
86
+ @property
87
+ def changed(self) -> bool:
88
+ return bool(self.added or self.updated or self.removed or self.mounts_changed)
89
+
90
+ def to_dict(self) -> dict[str, Any]:
91
+ return {
92
+ "added": self.added,
93
+ "updated": self.updated,
94
+ "removed": self.removed,
95
+ "touched": self.touched,
96
+ "mounts_changed": self.mounts_changed,
97
+ }
98
+
99
+
100
+ @dataclass
101
+ class DiskState:
102
+ files: dict[tuple[str, str], tuple[int, int, Path]] # (mount, path) -> (mtime_ns, size, abs path)
103
+
104
+
105
+ @dataclass
106
+ class Hit:
107
+ """One search match: the card row plus its ranking signals."""
108
+
109
+ row: sqlite3.Row
110
+ score: float # -bm25(...) > 0 (0.0 for filter-only listings)
111
+ coverage: float # IDF-weighted share of the query matched, in [0, 1] (reported, not used to order)
112
+ phrases: int # unspaced query runs (3+ units) matched verbatim (reported, not used to order)
113
+
114
+ def rank_key(self) -> tuple:
115
+ """BM25 score, then mount precedence, then address (deterministic)."""
116
+ r = self.row
117
+ return (-self.score, -r["precedence"], r["mount"], r["path"])
118
+
119
+
120
+ FRONTMATTER_CODES = {None: 0, "toml": 1, "yaml": 2} # cards.has_frontmatter
121
+
122
+
123
+ def index_signature() -> str:
124
+ """What derived index content depends on besides the files: text analysis and card parsing.
125
+
126
+ A different value re-indexes every card (telemetry is kept).
127
+ """
128
+ return f"{analyzer_signature()}|parser-{PARSER_VERSION}"
129
+
130
+
131
+ def mount_signature(table: MountTable) -> str:
132
+ return json.dumps(
133
+ [[m.alias, str(m.root), m.precedence, m.role, m.kind] for m in table.all()], sort_keys=True
134
+ )
135
+
136
+
137
+ class Index:
138
+ def __init__(self, db_path: Path) -> None:
139
+ self.db_path = db_path
140
+ self.conn = self._connect()
141
+
142
+ # ------------------------------------------------------------------
143
+ # connection / schema
144
+ # ------------------------------------------------------------------
145
+ def _connect(self) -> sqlite3.Connection:
146
+ self.db_path.parent.mkdir(parents=True, exist_ok=True)
147
+ conn = None
148
+ try:
149
+ conn = self._open()
150
+ version = self._schema_version(conn)
151
+ if version not in (None, SCHEMA_VERSION):
152
+ conn.close()
153
+ self._delete_db()
154
+ conn = self._open()
155
+ except sqlite3.DatabaseError:
156
+ if conn is not None:
157
+ conn.close()
158
+ self._delete_db()
159
+ conn = self._open()
160
+ conn.executescript(_SCHEMA)
161
+ conn.execute("INSERT OR IGNORE INTO meta(key, value) VALUES('schema_version', ?)", (SCHEMA_VERSION,))
162
+ conn.commit()
163
+ return conn
164
+
165
+ def _open(self) -> sqlite3.Connection:
166
+ conn = sqlite3.connect(self.db_path, timeout=5.0, check_same_thread=False, isolation_level=None)
167
+ try:
168
+ conn.row_factory = sqlite3.Row
169
+ conn.execute("PRAGMA journal_mode=WAL")
170
+ conn.execute("PRAGMA synchronous=NORMAL")
171
+ conn.execute("PRAGMA foreign_keys=OFF")
172
+ conn.execute("SELECT 1").fetchone() # surfaces 'file is not a database'
173
+ except BaseException:
174
+ # Windows cannot unlink a corrupt database while this handle is open.
175
+ conn.close()
176
+ raise
177
+ return conn
178
+
179
+ @staticmethod
180
+ def _schema_version(conn: sqlite3.Connection) -> str | None:
181
+ row = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='meta'").fetchone()
182
+ if row is None:
183
+ return None
184
+ row = conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone()
185
+ return row[0] if row else None
186
+
187
+ def _delete_db(self) -> None:
188
+ for suffix in ("", "-wal", "-shm", "-journal"):
189
+ try:
190
+ os.unlink(str(self.db_path) + suffix)
191
+ except FileNotFoundError:
192
+ pass
193
+
194
+ def close(self) -> None:
195
+ try:
196
+ self.conn.close()
197
+ except sqlite3.Error:
198
+ pass
199
+
200
+ def _meta(self, key: str) -> str | None:
201
+ row = self.conn.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone()
202
+ return row[0] if row else None
203
+
204
+ def _set_meta(self, key: str, value: str) -> None:
205
+ self.conn.execute("INSERT OR REPLACE INTO meta(key, value) VALUES(?, ?)", (key, value))
206
+
207
+ # ------------------------------------------------------------------
208
+ # disk scanning & freshness
209
+ # ------------------------------------------------------------------
210
+ @staticmethod
211
+ def scan(table: MountTable, exclude: Sequence[Path] = ()) -> DiskState:
212
+ files: dict[tuple[str, str], tuple[int, int, Path]] = {}
213
+ mounts = table.all()
214
+ for m in mounts:
215
+ nested = tuple(o.root for o in mounts if o.alias != m.alias and m.root in o.root.parents)
216
+ for rel, st in iter_markdown(m.root, exclude=tuple(exclude) + nested):
217
+ files[(m.alias, rel)] = (st.st_mtime_ns, st.st_size, m.root / (rel + ".md"))
218
+ return DiskState(files)
219
+
220
+ def indexed_stats(self) -> dict[tuple[str, str], tuple[int, int, str]]:
221
+ rows = self.conn.execute("SELECT mount, path, mtime_ns, size, sha256 FROM cards")
222
+ return {(r[0], r[1]): (r[2], r[3], r[4]) for r in rows}
223
+
224
+ def is_stale(self, table: MountTable, exclude: Sequence[Path] = ()) -> bool:
225
+ if self._meta("analyzer") != index_signature():
226
+ return True
227
+ if self._meta("mounts_signature") != mount_signature(table):
228
+ return True
229
+ disk = self.scan(table, exclude).files
230
+ indexed = self.indexed_stats()
231
+ if disk.keys() != indexed.keys():
232
+ return True
233
+ return any(indexed[k][:2] != (v[0], v[1]) for k, v in disk.items())
234
+
235
+ # ------------------------------------------------------------------
236
+ # sync
237
+ # ------------------------------------------------------------------
238
+ def sync(self, table: MountTable, exclude: Sequence[Path] = (), verify: bool = False) -> SyncStats:
239
+ """Incrementally bring the index in line with disk in one transaction.
240
+
241
+ With ``verify=True`` every file is re-hashed (catching edits that preserved
242
+ both mtime and size).
243
+ """
244
+ stats = SyncStats()
245
+ conn = self.conn
246
+ conn.execute("BEGIN IMMEDIATE")
247
+ try:
248
+ analyzer = index_signature()
249
+ if self._meta("analyzer") != analyzer:
250
+ # Text analysis or card parsing changed (MindSlice upgrade or a new Unicode
251
+ # database): every card must be re-analyzed. Telemetry is kept.
252
+ self._reset_derived()
253
+ self._set_meta("analyzer", analyzer)
254
+ signature = mount_signature(table)
255
+ if self._meta("mounts_signature") != signature:
256
+ stats.mounts_changed = True
257
+ current = {m.alias: m for m in table.all()}
258
+ for row in conn.execute("SELECT alias, root FROM mounts").fetchall():
259
+ m = current.get(row["alias"])
260
+ if m is None or str(m.root) != row["root"]:
261
+ stats.removed += self._delete_mount_rows(row["alias"])
262
+ conn.execute("DELETE FROM mounts")
263
+ conn.executemany(
264
+ "INSERT INTO mounts(alias, root, precedence, role, kind) VALUES(?, ?, ?, ?, ?)",
265
+ [(m.alias, str(m.root), m.precedence, m.role, m.kind) for m in table.all()],
266
+ )
267
+ self._set_meta("mounts_signature", signature)
268
+
269
+ disk = self.scan(table, exclude).files
270
+ indexed = self.indexed_stats()
271
+ for key in indexed.keys() - disk.keys():
272
+ self._delete_card(*key)
273
+ stats.removed += 1
274
+ stats.changed_cards.append(key)
275
+ for key, (mtime_ns, size, abspath) in disk.items():
276
+ prev = indexed.get(key)
277
+ if prev is not None and not verify and prev[:2] == (mtime_ns, size):
278
+ continue
279
+ try:
280
+ data = abspath.read_bytes()
281
+ except OSError:
282
+ continue
283
+ digest = sha256_bytes(data)
284
+ if prev is not None and prev[2] == digest:
285
+ conn.execute(
286
+ "UPDATE cards SET mtime_ns=?, size=? WHERE mount=? AND path=?",
287
+ (mtime_ns, size, key[0], key[1]),
288
+ )
289
+ stats.touched += 1
290
+ continue
291
+ self._upsert_card(key[0], key[1], data, digest, mtime_ns, size)
292
+ stats.changed_cards.append(key)
293
+ if prev is None:
294
+ stats.added += 1
295
+ else:
296
+ stats.updated += 1
297
+ if stats.changed:
298
+ self._resolve_all_links()
299
+ conn.execute("COMMIT")
300
+ except BaseException:
301
+ conn.execute("ROLLBACK")
302
+ raise
303
+ return stats
304
+
305
+ def rebuild(self, table: MountTable, exclude: Sequence[Path] = ()) -> SyncStats:
306
+ """Drop all reconstructible state and re-index from disk with full sha256 verification."""
307
+ conn = self.conn
308
+ conn.execute("BEGIN IMMEDIATE")
309
+ try:
310
+ self._reset_derived()
311
+ conn.execute("COMMIT")
312
+ except BaseException:
313
+ conn.execute("ROLLBACK")
314
+ raise
315
+ return self.sync(table, exclude, verify=True)
316
+
317
+ def _reset_derived(self) -> None:
318
+ """Drop everything reconstructible from disk (telemetry and schema version survive)."""
319
+ for name in _DERIVED_TABLES:
320
+ self.conn.execute(f"DELETE FROM {name}")
321
+ self.conn.execute("DELETE FROM meta WHERE key <> 'schema_version'")
322
+
323
+ def _delete_mount_rows(self, alias: str) -> int:
324
+ ids = [r[0] for r in self.conn.execute("SELECT id FROM cards WHERE mount=?", (alias,))]
325
+ self.conn.executemany("DELETE FROM cards_fts WHERE rowid=?", [(i,) for i in ids])
326
+ for tbl in ("cards", "card_aliases", "card_tags"):
327
+ self.conn.execute(f"DELETE FROM {tbl} WHERE mount=?", (alias,))
328
+ self.conn.execute("DELETE FROM links WHERE src_mount=?", (alias,))
329
+ return len(ids)
330
+
331
+ def _delete_card(self, mount: str, path: str) -> None:
332
+ row = self.conn.execute("SELECT id FROM cards WHERE mount=? AND path=?", (mount, path)).fetchone()
333
+ if row is not None:
334
+ self.conn.execute("DELETE FROM cards_fts WHERE rowid=?", (row[0],))
335
+ self.conn.execute("DELETE FROM cards WHERE mount=? AND path=?", (mount, path))
336
+ self.conn.execute("DELETE FROM card_aliases WHERE mount=? AND path=?", (mount, path))
337
+ self.conn.execute("DELETE FROM card_tags WHERE mount=? AND path=?", (mount, path))
338
+ self.conn.execute("DELETE FROM links WHERE src_mount=? AND src_path=?", (mount, path))
339
+
340
+ def _upsert_card(self, mount: str, path: str, data: bytes, digest: str, mtime_ns: int, size: int) -> None:
341
+ self._delete_card(mount, path)
342
+ text = data.decode("utf-8", errors="replace")
343
+ parsed = parse_card(text)
344
+ title = parsed.display_title(path)
345
+ summary, body = parsed.index_parts() # plain Markdown: the lead paragraph is the summary
346
+ preview, _ = preview_text(summary, body, limit=10_000)
347
+ lines = text.count("\n") + (0 if text.endswith("\n") or not text else 1)
348
+ cur = self.conn.execute(
349
+ """INSERT INTO cards(mount, path, title, aliases, tags, extends, supersedes, summary, preview,
350
+ sha256, mtime_ns, size, lines, is_overview, has_frontmatter, load, errors, warnings)
351
+ VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
352
+ (
353
+ mount,
354
+ path,
355
+ title,
356
+ json.dumps(parsed.aliases, ensure_ascii=False),
357
+ json.dumps(parsed.tags, ensure_ascii=False),
358
+ json.dumps(parsed.extends, ensure_ascii=False),
359
+ json.dumps(parsed.supersedes, ensure_ascii=False),
360
+ summary,
361
+ preview,
362
+ digest,
363
+ mtime_ns,
364
+ size,
365
+ lines,
366
+ 1 if path == OVERVIEW_PATH else 0,
367
+ FRONTMATTER_CODES.get(parsed.frontmatter_format, 0),
368
+ parsed.load,
369
+ json.dumps(parsed.errors, ensure_ascii=False),
370
+ json.dumps(parsed.warnings, ensure_ascii=False),
371
+ ),
372
+ )
373
+ card_id = cur.lastrowid
374
+ alias_text = " ".join(parsed.aliases + [path.replace("/", " ")])
375
+ fields = (title, alias_text, " ".join(parsed.tags), summary, body)
376
+ self.conn.execute(
377
+ "INSERT INTO cards_fts(rowid, title, aliases, tags, summary, body) VALUES(?, ?, ?, ?, ?, ?)",
378
+ (card_id, *(index_text(t) for t in fields)),
379
+ )
380
+ self.conn.executemany(
381
+ "INSERT INTO card_aliases(mount, path, alias) VALUES(?, ?, ?)",
382
+ [(mount, path, k) for k in dict.fromkeys(fold(a) for a in parsed.aliases) if k],
383
+ )
384
+ self.conn.executemany(
385
+ "INSERT INTO card_tags(mount, path, tag) VALUES(?, ?, ?)",
386
+ [(mount, path, k) for k in dict.fromkeys(fold(t) for t in parsed.tags) if k],
387
+ )
388
+ refs: list[tuple[str, int, str]] = []
389
+ for kind, values in (("extends", parsed.extends), ("supersedes", parsed.supersedes)):
390
+ refs += [(kind, i, v) for i, v in enumerate(values)]
391
+ refs += [("wikilink", i, v) for i, v in enumerate(parsed.wikilink_targets())]
392
+ self.conn.executemany(
393
+ "INSERT INTO links(src_mount, src_path, kind, ord, raw) VALUES(?, ?, ?, ?, ?)",
394
+ [(mount, path, kind, i, raw) for kind, i, raw in refs],
395
+ )
396
+
397
+ def catalogs(self) -> dict[str, MountCatalog]:
398
+ cats: dict[str, MountCatalog] = {}
399
+ for row in self.conn.execute("SELECT alias FROM mounts"):
400
+ cats[row[0]] = MountCatalog()
401
+ for row in self.conn.execute("SELECT mount, path, aliases FROM cards ORDER BY mount, path"):
402
+ cats.setdefault(row[0], MountCatalog()).add(row[1], json.loads(row[2]))
403
+ return cats
404
+
405
+ def _resolve_all_links(self) -> None:
406
+ cats = self.catalogs()
407
+ updates = []
408
+ for row in self.conn.execute("SELECT id, src_mount, raw FROM links").fetchall():
409
+ res = resolve_ref(row["raw"], row["src_mount"], cats)
410
+ updates.append((res.mount, res.path, 1 if res.resolved else 0, row["id"]))
411
+ self.conn.executemany("UPDATE links SET dst_mount=?, dst_path=?, resolved=? WHERE id=?", updates)
412
+
413
+ # ------------------------------------------------------------------
414
+ # queries
415
+ # ------------------------------------------------------------------
416
+ def mount_rows(self) -> list[sqlite3.Row]:
417
+ return self.conn.execute("SELECT * FROM mounts ORDER BY precedence DESC, alias").fetchall()
418
+
419
+ def card(self, mount: str, path: str) -> sqlite3.Row | None:
420
+ return self.conn.execute(
421
+ """SELECT c.*, m.precedence, m.role FROM cards c JOIN mounts m ON m.alias = c.mount
422
+ WHERE c.mount=? AND c.path=?""",
423
+ (mount, path),
424
+ ).fetchone()
425
+
426
+ def cards_at_path(self, path: str) -> list[sqlite3.Row]:
427
+ return self.conn.execute(
428
+ """SELECT c.*, m.precedence, m.role FROM cards c JOIN mounts m ON m.alias = c.mount
429
+ WHERE c.path=? ORDER BY m.precedence DESC, c.mount""",
430
+ (path,),
431
+ ).fetchall()
432
+
433
+ def cards_with_alias(self, alias: str, mount: str | None = None) -> list[sqlite3.Row]:
434
+ sql = """SELECT c.*, m.precedence, m.role FROM card_aliases a
435
+ JOIN cards c ON c.mount = a.mount AND c.path = a.path
436
+ JOIN mounts m ON m.alias = c.mount WHERE a.alias=?"""
437
+ args: list[Any] = [fold(alias)]
438
+ if mount is not None:
439
+ sql += " AND a.mount=?"
440
+ args.append(mount)
441
+ sql += " ORDER BY m.precedence DESC, c.mount, c.path"
442
+ return self.conn.execute(sql, args).fetchall()
443
+
444
+ def always_cards(self) -> list[sqlite3.Row]:
445
+ """Cards with ``load = "always"``, base layers first (precedence ascending), then mount, path."""
446
+ return self.conn.execute(
447
+ """SELECT c.*, m.precedence, m.role FROM cards c JOIN mounts m ON m.alias = c.mount
448
+ WHERE c.load = 'always' ORDER BY m.precedence, c.mount, c.path"""
449
+ ).fetchall()
450
+
451
+ def all_cards(self, mount: str | None = None) -> list[sqlite3.Row]:
452
+ sql = "SELECT c.*, m.precedence, m.role FROM cards c JOIN mounts m ON m.alias = c.mount"
453
+ args: tuple[Any, ...] = ()
454
+ if mount is not None:
455
+ sql += " WHERE c.mount=?"
456
+ args = (mount,)
457
+ return self.conn.execute(sql + " ORDER BY m.precedence DESC, c.mount, c.path", args).fetchall()
458
+
459
+ def search(
460
+ self, query: str, sources: Iterable[str] | None = None, tags: Iterable[str] | None = None
461
+ ) -> list[Hit]:
462
+ """All matches (overviews excluded), best first: ``score = -bm25(...)``, then precedence.
463
+
464
+ Each hit also carries ``coverage``: the IDF-weighted share of the query it matches (an
465
+ unspaced run earns the weighted share of its bigrams, or full credit when matched
466
+ verbatim). It is reported for callers and does not order results: on the retrieval
467
+ evaluation, coverage-first ordering lost to BM25 order (evals/retrieval/PLAN.md).
468
+ An empty query with ``tags`` lists that facet (score and coverage 0).
469
+ """
470
+ plan = analyze_query(query)
471
+ where, args = self._scope(sources, tags)
472
+ if plan.units:
473
+ weights = ", ".join(str(w) for w in BM25_WEIGHTS)
474
+ sql = f"""SELECT c.*, m.precedence, m.role, -bm25(cards_fts, {weights}) AS score
475
+ FROM cards_fts JOIN cards c ON c.id = cards_fts.rowid
476
+ JOIN mounts m ON m.alias = c.mount"""
477
+ where.insert(0, "cards_fts MATCH ?")
478
+ args.insert(0, plan.match_expression) # its placeholder comes first, before the scope clauses
479
+ elif (query or "").strip():
480
+ return [] # punctuation-only query: nothing searchable
481
+ else:
482
+ sql = """SELECT c.*, m.precedence, m.role, 0.0 AS score FROM cards c
483
+ JOIN mounts m ON m.alias = c.mount"""
484
+ sql += " WHERE " + " AND ".join(where)
485
+ rows = self.conn.execute(sql, args).fetchall()
486
+ hits = self._rank(plan, rows)
487
+ hits.sort(key=Hit.rank_key)
488
+ return hits
489
+
490
+ @staticmethod
491
+ def _scope(sources: Iterable[str] | None, tags: Iterable[str] | None) -> tuple[list[str], list[Any]]:
492
+ """WHERE clauses (over ``cards c``) and their arguments: the searchable cards in scope."""
493
+ where: list[str] = ["c.is_overview = 0"]
494
+ args: list[Any] = []
495
+ source_list = list(sources or [])
496
+ if source_list:
497
+ where.append(f"c.mount IN ({', '.join('?' * len(source_list))})")
498
+ args += source_list
499
+ tag_list = sorted({fold(t) for t in (tags or []) if fold(t)})
500
+ if tag_list:
501
+ where.append(
502
+ f"""(SELECT COUNT(DISTINCT t.tag) FROM card_tags t WHERE t.mount = c.mount AND t.path = c.path
503
+ AND t.tag IN ({', '.join('?' * len(tag_list))})) = ?"""
504
+ )
505
+ args += tag_list + [len(tag_list)]
506
+ return where, args
507
+
508
+ def unmatched_terms(self, query: str, hits: Sequence[Hit]) -> list[str]:
509
+ """Query words (an unspaced run counts as one) that no card in scope contains.
510
+
511
+ ``hits`` must be ``search()``'s result for the same query and scope: a word is unmatched
512
+ when none of its terms occurs in any hit. It is a plain fact about the corpus, with no
513
+ heuristics, in every script; it changes and reorders nothing. (Spelling suggestions were
514
+ tried and removed: evals/retrieval/PLAN.md, "Decision: no did_you_mean".)
515
+ """
516
+ plan = analyze_query(query)
517
+ ids = {h.row["id"] for h in hits}
518
+ if not ids:
519
+ return [u.text for u in plan.units]
520
+ if len(plan.units) == 1:
521
+ return []
522
+ return [u.text for u in plan.units if not any(self._matching_ids(t) & ids for t in u.terms)]
523
+
524
+ def _matching_ids(self, expression: str) -> set[int]:
525
+ return {r[0] for r in self.conn.execute("SELECT rowid FROM cards_fts WHERE cards_fts MATCH ?", (expression,))}
526
+
527
+ def _rank(self, plan: QueryPlan, rows: list[sqlite3.Row]) -> list[Hit]:
528
+ if not plan.units:
529
+ return [Hit(r, 0.0, 0.0, 0) for r in rows]
530
+ if len(plan.units) == 1 and len(plan.units[0].terms) == 1:
531
+ return [Hit(r, float(r["score"]), 1.0, 0) for r in rows] # every match matches everything
532
+ term_ids = {t: self._matching_ids(t) for t in plan.terms}
533
+ phrase_ids = {p: self._matching_ids(p) for p in plan.phrases}
534
+ n_cards = self.conn.execute("SELECT COUNT(*) FROM cards").fetchone()[0]
535
+ # Smoothed BM25 IDF (always positive, also in tiny corpora where FTS5 clamps its own to ~0).
536
+ idf = {t: math.log(1 + (n_cards - len(ids) + 0.5) / (len(ids) + 0.5)) for t, ids in term_ids.items()}
537
+ unit_weight = [sum(idf[t] for t in u.terms) / len(u.terms) for u in plan.units]
538
+ total = sum(unit_weight)
539
+ hits = []
540
+ for r in rows:
541
+ rid = r["id"]
542
+ credit = 0.0
543
+ phrases = 0
544
+ for unit, weight in zip(plan.units, unit_weight):
545
+ if unit.phrase and rid in phrase_ids[unit.phrase]:
546
+ credit += weight
547
+ phrases += 1
548
+ continue
549
+ matched = sum(idf[t] for t in unit.terms if rid in term_ids[t])
550
+ credit += weight * matched / sum(idf[t] for t in unit.terms)
551
+ hits.append(Hit(r, float(r["score"]), credit / total if total else 0.0, phrases))
552
+ return hits
553
+
554
+ def card_has_tags(self, mount: str, path: str, tags: Iterable[str]) -> bool:
555
+ wanted = {fold(t) for t in tags if fold(t)}
556
+ if not wanted:
557
+ return True
558
+ have = {r[0] for r in self.conn.execute("SELECT tag FROM card_tags WHERE mount=? AND path=?", (mount, path))}
559
+ return wanted <= have
560
+
561
+ def inbound(self, kind: str, mount: str, path: str, recency: bool = False) -> list[sqlite3.Row]:
562
+ """Cards whose ``kind`` reference resolves to ``(mount, path)``.
563
+
564
+ Ordered by precedence, then address (deterministic). ``recency=True`` orders by
565
+ precedence then newest first (the design's order for ``extensions``).
566
+ """
567
+ order = "m.precedence DESC, c.mtime_ns DESC," if recency else "m.precedence DESC,"
568
+ return self.conn.execute(
569
+ """SELECT DISTINCT l.src_mount, l.src_path, c.mtime_ns, m.precedence, m.role
570
+ FROM links l JOIN cards c ON c.mount = l.src_mount AND c.path = l.src_path
571
+ JOIN mounts m ON m.alias = l.src_mount
572
+ WHERE l.kind=? AND l.dst_mount=? AND l.dst_path=? AND l.resolved=1
573
+ AND NOT (l.src_mount = l.dst_mount AND l.src_path = l.dst_path)
574
+ ORDER BY """ + order + " l.src_mount, l.src_path",
575
+ (kind, mount, path),
576
+ ).fetchall()
577
+
578
+ def dangling_inbound(self, kind: str, mount: str, path: str) -> list[sqlite3.Row]:
579
+ """Cards whose ``kind`` reference names ``(mount, path)`` by path but no longer resolves (a deleted target)."""
580
+ return self.conn.execute(
581
+ """SELECT DISTINCT l.src_mount, l.src_path, c.mtime_ns, m.precedence, m.role
582
+ FROM links l JOIN cards c ON c.mount = l.src_mount AND c.path = l.src_path
583
+ JOIN mounts m ON m.alias = l.src_mount
584
+ WHERE l.kind=? AND l.dst_mount=? AND l.dst_path=? AND l.resolved=0
585
+ ORDER BY m.precedence DESC, l.src_mount, l.src_path""",
586
+ (kind, mount, path),
587
+ ).fetchall()
588
+
589
+ def outbound(self, mount: str, path: str, kind: str | None = None) -> list[sqlite3.Row]:
590
+ sql = "SELECT kind, ord, raw, dst_mount, dst_path, resolved FROM links WHERE src_mount=? AND src_path=?"
591
+ args: list[Any] = [mount, path]
592
+ if kind is not None:
593
+ sql += " AND kind=?"
594
+ args.append(kind)
595
+ return self.conn.execute(sql + " ORDER BY kind, ord", args).fetchall()
596
+
597
+ def lineage_edges(self) -> list[sqlite3.Row]:
598
+ return self.conn.execute(
599
+ """SELECT src_mount, src_path, kind, raw, dst_mount, dst_path, resolved FROM links
600
+ WHERE kind IN ('extends', 'supersedes')"""
601
+ ).fetchall()
602
+
603
+ def folder_counts(self, mount: str) -> tuple[list[tuple[str, int]], int, int]:
604
+ """``([(folder, card_count)...], root_card_count, total_cards)`` for one mount (overview excluded)."""
605
+ folders: dict[str, int] = {}
606
+ root = 0
607
+ total = 0
608
+ for row in self.conn.execute("SELECT path FROM cards WHERE mount=? AND is_overview=0", (mount,)):
609
+ total += 1
610
+ head, sep, _ = row[0].partition("/")
611
+ if sep:
612
+ folders[head] = folders.get(head, 0) + 1
613
+ else:
614
+ root += 1
615
+ ordered = sorted(folders.items(), key=lambda kv: (-kv[1], kv[0]))
616
+ return ordered, root, total
617
+
618
+ # ------------------------------------------------------------------
619
+ # telemetry (best effort)
620
+ # ------------------------------------------------------------------
621
+ def bump_read(self, mount: str, path: str) -> tuple[int, float] | None:
622
+ now = time.time()
623
+ try:
624
+ self.conn.execute(
625
+ """INSERT INTO telemetry(mount, path, read_count, last_read_at) VALUES(?, ?, 1, ?)
626
+ ON CONFLICT(mount, path) DO UPDATE SET read_count = read_count + 1, last_read_at = excluded.last_read_at""",
627
+ (mount, path, now),
628
+ )
629
+ row = self.conn.execute(
630
+ "SELECT read_count, last_read_at FROM telemetry WHERE mount=? AND path=?", (mount, path)
631
+ ).fetchone()
632
+ return (row[0], row[1]) if row else None
633
+ except sqlite3.Error:
634
+ return None