mindslice 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mindslice/__init__.py +58 -0
- mindslice/__main__.py +5 -0
- mindslice/baking/__init__.py +1 -0
- mindslice/baking/archive.py +66 -0
- mindslice/baking/bake.py +299 -0
- mindslice/baking/bake_batch.py +537 -0
- mindslice/baking/slice_transfer.py +265 -0
- mindslice/errors.py +104 -0
- mindslice/interfaces/__init__.py +1 -0
- mindslice/interfaces/bake_review.py +230 -0
- mindslice/interfaces/cli.py +392 -0
- mindslice/interfaces/cli_addressing.py +52 -0
- mindslice/interfaces/doctor.py +184 -0
- mindslice/interfaces/mcp_server.py +536 -0
- mindslice/interfaces/render.py +346 -0
- mindslice/interfaces/staging_review.py +92 -0
- mindslice/model/__init__.py +1 -0
- mindslice/model/addressing.py +251 -0
- mindslice/model/card.py +641 -0
- mindslice/model/lineage.py +228 -0
- mindslice/model/text.py +19 -0
- mindslice/model/tomlfmt.py +129 -0
- mindslice/operations/__init__.py +1 -0
- mindslice/operations/audit.py +195 -0
- mindslice/operations/boot.py +183 -0
- mindslice/operations/impact.py +98 -0
- mindslice/operations/recovery.py +113 -0
- mindslice/operations/staging.py +641 -0
- mindslice/operations/status.py +137 -0
- mindslice/operations/validation.py +250 -0
- mindslice/operations/writer.py +481 -0
- mindslice/retrieval/__init__.py +1 -0
- mindslice/retrieval/index.py +634 -0
- mindslice/retrieval/lexical.py +215 -0
- mindslice/retrieval/query.py +593 -0
- mindslice/storage/__init__.py +1 -0
- mindslice/storage/card_io.py +14 -0
- mindslice/storage/config.py +192 -0
- mindslice/storage/fsutil.py +141 -0
- mindslice/storage/history.py +182 -0
- mindslice/storage/locking.py +125 -0
- mindslice/storage/snapshots.py +262 -0
- mindslice/workspace.py +385 -0
- mindslice-0.1.0.dist-info/METADATA +98 -0
- mindslice-0.1.0.dist-info/RECORD +50 -0
- mindslice-0.1.0.dist-info/WHEEL +5 -0
- mindslice-0.1.0.dist-info/entry_points.txt +2 -0
- mindslice-0.1.0.dist-info/licenses/LICENSE +202 -0
- mindslice-0.1.0.dist-info/licenses/NOTICE +22 -0
- mindslice-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,634 @@
|
|
|
1
|
+
"""Disposable SQLite acceleration cache (``.cache/index.sqlite``).
|
|
2
|
+
|
|
3
|
+
Everything except telemetry is reconstructible from the Markdown on disk:
|
|
4
|
+
|
|
5
|
+
* ``cards`` one row per card file (stat, sha256, parsed metadata, preview)
|
|
6
|
+
* ``cards_fts`` FTS5 table over ``title, aliases, tags, summary, body``
|
|
7
|
+
* ``links`` raw lineage / wikilink references plus their current resolution
|
|
8
|
+
* ``mounts`` snapshot of the mount table the index was built against
|
|
9
|
+
* ``telemetry`` best-effort ``read_count`` / ``last_read_at`` (survives rebuilds,
|
|
10
|
+
but is lost if the cache file is deleted)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import math
|
|
17
|
+
import os
|
|
18
|
+
import sqlite3
|
|
19
|
+
import time
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Iterable, Sequence
|
|
23
|
+
|
|
24
|
+
from ..model.addressing import MountCatalog, resolve_ref
|
|
25
|
+
from ..model.card import PARSER_VERSION, parse_card, preview_text, sha256_bytes
|
|
26
|
+
from ..storage.config import OVERVIEW_PATH, Mount, MountTable
|
|
27
|
+
from ..storage.fsutil import iter_markdown
|
|
28
|
+
from .lexical import QueryPlan, analyze_query, analyzer_signature, fold, index_text
|
|
29
|
+
|
|
30
|
+
SCHEMA_VERSION = "2" # table layout ("2": cards.load); text-analysis and parsing changes are tracked by index_signature()
|
|
31
|
+
BM25_WEIGHTS = (10.0, 5.0, 3.0, 2.0, 1.0) # title, aliases, tags, summary, body
|
|
32
|
+
TOKENIZER = "porter unicode61 remove_diacritics 2"
|
|
33
|
+
LINK_KINDS = ("extends", "supersedes", "wikilink")
|
|
34
|
+
|
|
35
|
+
_SCHEMA = f"""
|
|
36
|
+
CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
37
|
+
CREATE TABLE IF NOT EXISTS mounts(
|
|
38
|
+
alias TEXT PRIMARY KEY, root TEXT NOT NULL, precedence INTEGER NOT NULL,
|
|
39
|
+
role TEXT NOT NULL, kind TEXT NOT NULL
|
|
40
|
+
);
|
|
41
|
+
CREATE TABLE IF NOT EXISTS cards(
|
|
42
|
+
id INTEGER PRIMARY KEY,
|
|
43
|
+
mount TEXT NOT NULL, path TEXT NOT NULL,
|
|
44
|
+
title TEXT NOT NULL, aliases TEXT NOT NULL, tags TEXT NOT NULL,
|
|
45
|
+
extends TEXT NOT NULL, supersedes TEXT NOT NULL,
|
|
46
|
+
summary TEXT NOT NULL, preview TEXT NOT NULL,
|
|
47
|
+
sha256 TEXT NOT NULL, mtime_ns INTEGER NOT NULL, size INTEGER NOT NULL,
|
|
48
|
+
lines INTEGER NOT NULL, is_overview INTEGER NOT NULL, has_frontmatter INTEGER NOT NULL,
|
|
49
|
+
load TEXT NOT NULL DEFAULT 'as-needed',
|
|
50
|
+
errors TEXT NOT NULL, warnings TEXT NOT NULL,
|
|
51
|
+
UNIQUE(mount, path)
|
|
52
|
+
);
|
|
53
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS cards_fts USING fts5(
|
|
54
|
+
title, aliases, tags, summary, body, tokenize = '{TOKENIZER}'
|
|
55
|
+
);
|
|
56
|
+
CREATE TABLE IF NOT EXISTS card_aliases(mount TEXT NOT NULL, path TEXT NOT NULL, alias TEXT NOT NULL);
|
|
57
|
+
CREATE INDEX IF NOT EXISTS card_aliases_idx ON card_aliases(mount, alias);
|
|
58
|
+
CREATE TABLE IF NOT EXISTS card_tags(mount TEXT NOT NULL, path TEXT NOT NULL, tag TEXT NOT NULL);
|
|
59
|
+
DROP TABLE IF EXISTS card_words; -- left by analyzer "2" caches (did_you_mean, since removed)
|
|
60
|
+
CREATE INDEX IF NOT EXISTS card_tags_idx ON card_tags(mount, path);
|
|
61
|
+
CREATE TABLE IF NOT EXISTS links(
|
|
62
|
+
id INTEGER PRIMARY KEY,
|
|
63
|
+
src_mount TEXT NOT NULL, src_path TEXT NOT NULL, kind TEXT NOT NULL, ord INTEGER NOT NULL,
|
|
64
|
+
raw TEXT NOT NULL, dst_mount TEXT, dst_path TEXT, resolved INTEGER NOT NULL DEFAULT 0
|
|
65
|
+
);
|
|
66
|
+
CREATE INDEX IF NOT EXISTS links_src_idx ON links(src_mount, src_path);
|
|
67
|
+
CREATE INDEX IF NOT EXISTS links_dst_idx ON links(kind, dst_mount, dst_path);
|
|
68
|
+
CREATE TABLE IF NOT EXISTS telemetry(
|
|
69
|
+
mount TEXT NOT NULL, path TEXT NOT NULL, read_count INTEGER NOT NULL DEFAULT 0,
|
|
70
|
+
last_read_at REAL, PRIMARY KEY(mount, path)
|
|
71
|
+
);
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
_DERIVED_TABLES = ("cards", "cards_fts", "card_aliases", "card_tags", "links", "mounts")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class SyncStats:
|
|
79
|
+
added: int = 0
|
|
80
|
+
updated: int = 0
|
|
81
|
+
removed: int = 0
|
|
82
|
+
touched: int = 0 # stat changed but content identical
|
|
83
|
+
mounts_changed: bool = False
|
|
84
|
+
changed_cards: list[tuple[str, str]] = field(default_factory=list)
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def changed(self) -> bool:
|
|
88
|
+
return bool(self.added or self.updated or self.removed or self.mounts_changed)
|
|
89
|
+
|
|
90
|
+
def to_dict(self) -> dict[str, Any]:
|
|
91
|
+
return {
|
|
92
|
+
"added": self.added,
|
|
93
|
+
"updated": self.updated,
|
|
94
|
+
"removed": self.removed,
|
|
95
|
+
"touched": self.touched,
|
|
96
|
+
"mounts_changed": self.mounts_changed,
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class DiskState:
|
|
102
|
+
files: dict[tuple[str, str], tuple[int, int, Path]] # (mount, path) -> (mtime_ns, size, abs path)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass
|
|
106
|
+
class Hit:
|
|
107
|
+
"""One search match: the card row plus its ranking signals."""
|
|
108
|
+
|
|
109
|
+
row: sqlite3.Row
|
|
110
|
+
score: float # -bm25(...) > 0 (0.0 for filter-only listings)
|
|
111
|
+
coverage: float # IDF-weighted share of the query matched, in [0, 1] (reported, not used to order)
|
|
112
|
+
phrases: int # unspaced query runs (3+ units) matched verbatim (reported, not used to order)
|
|
113
|
+
|
|
114
|
+
def rank_key(self) -> tuple:
|
|
115
|
+
"""BM25 score, then mount precedence, then address (deterministic)."""
|
|
116
|
+
r = self.row
|
|
117
|
+
return (-self.score, -r["precedence"], r["mount"], r["path"])
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
FRONTMATTER_CODES = {None: 0, "toml": 1, "yaml": 2} # cards.has_frontmatter
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def index_signature() -> str:
|
|
124
|
+
"""What derived index content depends on besides the files: text analysis and card parsing.
|
|
125
|
+
|
|
126
|
+
A different value re-indexes every card (telemetry is kept).
|
|
127
|
+
"""
|
|
128
|
+
return f"{analyzer_signature()}|parser-{PARSER_VERSION}"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def mount_signature(table: MountTable) -> str:
|
|
132
|
+
return json.dumps(
|
|
133
|
+
[[m.alias, str(m.root), m.precedence, m.role, m.kind] for m in table.all()], sort_keys=True
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class Index:
|
|
138
|
+
def __init__(self, db_path: Path) -> None:
|
|
139
|
+
self.db_path = db_path
|
|
140
|
+
self.conn = self._connect()
|
|
141
|
+
|
|
142
|
+
# ------------------------------------------------------------------
|
|
143
|
+
# connection / schema
|
|
144
|
+
# ------------------------------------------------------------------
|
|
145
|
+
def _connect(self) -> sqlite3.Connection:
|
|
146
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
147
|
+
conn = None
|
|
148
|
+
try:
|
|
149
|
+
conn = self._open()
|
|
150
|
+
version = self._schema_version(conn)
|
|
151
|
+
if version not in (None, SCHEMA_VERSION):
|
|
152
|
+
conn.close()
|
|
153
|
+
self._delete_db()
|
|
154
|
+
conn = self._open()
|
|
155
|
+
except sqlite3.DatabaseError:
|
|
156
|
+
if conn is not None:
|
|
157
|
+
conn.close()
|
|
158
|
+
self._delete_db()
|
|
159
|
+
conn = self._open()
|
|
160
|
+
conn.executescript(_SCHEMA)
|
|
161
|
+
conn.execute("INSERT OR IGNORE INTO meta(key, value) VALUES('schema_version', ?)", (SCHEMA_VERSION,))
|
|
162
|
+
conn.commit()
|
|
163
|
+
return conn
|
|
164
|
+
|
|
165
|
+
def _open(self) -> sqlite3.Connection:
|
|
166
|
+
conn = sqlite3.connect(self.db_path, timeout=5.0, check_same_thread=False, isolation_level=None)
|
|
167
|
+
try:
|
|
168
|
+
conn.row_factory = sqlite3.Row
|
|
169
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
170
|
+
conn.execute("PRAGMA synchronous=NORMAL")
|
|
171
|
+
conn.execute("PRAGMA foreign_keys=OFF")
|
|
172
|
+
conn.execute("SELECT 1").fetchone() # surfaces 'file is not a database'
|
|
173
|
+
except BaseException:
|
|
174
|
+
# Windows cannot unlink a corrupt database while this handle is open.
|
|
175
|
+
conn.close()
|
|
176
|
+
raise
|
|
177
|
+
return conn
|
|
178
|
+
|
|
179
|
+
@staticmethod
|
|
180
|
+
def _schema_version(conn: sqlite3.Connection) -> str | None:
|
|
181
|
+
row = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='meta'").fetchone()
|
|
182
|
+
if row is None:
|
|
183
|
+
return None
|
|
184
|
+
row = conn.execute("SELECT value FROM meta WHERE key='schema_version'").fetchone()
|
|
185
|
+
return row[0] if row else None
|
|
186
|
+
|
|
187
|
+
def _delete_db(self) -> None:
|
|
188
|
+
for suffix in ("", "-wal", "-shm", "-journal"):
|
|
189
|
+
try:
|
|
190
|
+
os.unlink(str(self.db_path) + suffix)
|
|
191
|
+
except FileNotFoundError:
|
|
192
|
+
pass
|
|
193
|
+
|
|
194
|
+
def close(self) -> None:
|
|
195
|
+
try:
|
|
196
|
+
self.conn.close()
|
|
197
|
+
except sqlite3.Error:
|
|
198
|
+
pass
|
|
199
|
+
|
|
200
|
+
def _meta(self, key: str) -> str | None:
|
|
201
|
+
row = self.conn.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone()
|
|
202
|
+
return row[0] if row else None
|
|
203
|
+
|
|
204
|
+
def _set_meta(self, key: str, value: str) -> None:
|
|
205
|
+
self.conn.execute("INSERT OR REPLACE INTO meta(key, value) VALUES(?, ?)", (key, value))
|
|
206
|
+
|
|
207
|
+
# ------------------------------------------------------------------
|
|
208
|
+
# disk scanning & freshness
|
|
209
|
+
# ------------------------------------------------------------------
|
|
210
|
+
@staticmethod
|
|
211
|
+
def scan(table: MountTable, exclude: Sequence[Path] = ()) -> DiskState:
|
|
212
|
+
files: dict[tuple[str, str], tuple[int, int, Path]] = {}
|
|
213
|
+
mounts = table.all()
|
|
214
|
+
for m in mounts:
|
|
215
|
+
nested = tuple(o.root for o in mounts if o.alias != m.alias and m.root in o.root.parents)
|
|
216
|
+
for rel, st in iter_markdown(m.root, exclude=tuple(exclude) + nested):
|
|
217
|
+
files[(m.alias, rel)] = (st.st_mtime_ns, st.st_size, m.root / (rel + ".md"))
|
|
218
|
+
return DiskState(files)
|
|
219
|
+
|
|
220
|
+
def indexed_stats(self) -> dict[tuple[str, str], tuple[int, int, str]]:
|
|
221
|
+
rows = self.conn.execute("SELECT mount, path, mtime_ns, size, sha256 FROM cards")
|
|
222
|
+
return {(r[0], r[1]): (r[2], r[3], r[4]) for r in rows}
|
|
223
|
+
|
|
224
|
+
def is_stale(self, table: MountTable, exclude: Sequence[Path] = ()) -> bool:
|
|
225
|
+
if self._meta("analyzer") != index_signature():
|
|
226
|
+
return True
|
|
227
|
+
if self._meta("mounts_signature") != mount_signature(table):
|
|
228
|
+
return True
|
|
229
|
+
disk = self.scan(table, exclude).files
|
|
230
|
+
indexed = self.indexed_stats()
|
|
231
|
+
if disk.keys() != indexed.keys():
|
|
232
|
+
return True
|
|
233
|
+
return any(indexed[k][:2] != (v[0], v[1]) for k, v in disk.items())
|
|
234
|
+
|
|
235
|
+
# ------------------------------------------------------------------
|
|
236
|
+
# sync
|
|
237
|
+
# ------------------------------------------------------------------
|
|
238
|
+
def sync(self, table: MountTable, exclude: Sequence[Path] = (), verify: bool = False) -> SyncStats:
|
|
239
|
+
"""Incrementally bring the index in line with disk in one transaction.
|
|
240
|
+
|
|
241
|
+
With ``verify=True`` every file is re-hashed (catching edits that preserved
|
|
242
|
+
both mtime and size).
|
|
243
|
+
"""
|
|
244
|
+
stats = SyncStats()
|
|
245
|
+
conn = self.conn
|
|
246
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
247
|
+
try:
|
|
248
|
+
analyzer = index_signature()
|
|
249
|
+
if self._meta("analyzer") != analyzer:
|
|
250
|
+
# Text analysis or card parsing changed (MindSlice upgrade or a new Unicode
|
|
251
|
+
# database): every card must be re-analyzed. Telemetry is kept.
|
|
252
|
+
self._reset_derived()
|
|
253
|
+
self._set_meta("analyzer", analyzer)
|
|
254
|
+
signature = mount_signature(table)
|
|
255
|
+
if self._meta("mounts_signature") != signature:
|
|
256
|
+
stats.mounts_changed = True
|
|
257
|
+
current = {m.alias: m for m in table.all()}
|
|
258
|
+
for row in conn.execute("SELECT alias, root FROM mounts").fetchall():
|
|
259
|
+
m = current.get(row["alias"])
|
|
260
|
+
if m is None or str(m.root) != row["root"]:
|
|
261
|
+
stats.removed += self._delete_mount_rows(row["alias"])
|
|
262
|
+
conn.execute("DELETE FROM mounts")
|
|
263
|
+
conn.executemany(
|
|
264
|
+
"INSERT INTO mounts(alias, root, precedence, role, kind) VALUES(?, ?, ?, ?, ?)",
|
|
265
|
+
[(m.alias, str(m.root), m.precedence, m.role, m.kind) for m in table.all()],
|
|
266
|
+
)
|
|
267
|
+
self._set_meta("mounts_signature", signature)
|
|
268
|
+
|
|
269
|
+
disk = self.scan(table, exclude).files
|
|
270
|
+
indexed = self.indexed_stats()
|
|
271
|
+
for key in indexed.keys() - disk.keys():
|
|
272
|
+
self._delete_card(*key)
|
|
273
|
+
stats.removed += 1
|
|
274
|
+
stats.changed_cards.append(key)
|
|
275
|
+
for key, (mtime_ns, size, abspath) in disk.items():
|
|
276
|
+
prev = indexed.get(key)
|
|
277
|
+
if prev is not None and not verify and prev[:2] == (mtime_ns, size):
|
|
278
|
+
continue
|
|
279
|
+
try:
|
|
280
|
+
data = abspath.read_bytes()
|
|
281
|
+
except OSError:
|
|
282
|
+
continue
|
|
283
|
+
digest = sha256_bytes(data)
|
|
284
|
+
if prev is not None and prev[2] == digest:
|
|
285
|
+
conn.execute(
|
|
286
|
+
"UPDATE cards SET mtime_ns=?, size=? WHERE mount=? AND path=?",
|
|
287
|
+
(mtime_ns, size, key[0], key[1]),
|
|
288
|
+
)
|
|
289
|
+
stats.touched += 1
|
|
290
|
+
continue
|
|
291
|
+
self._upsert_card(key[0], key[1], data, digest, mtime_ns, size)
|
|
292
|
+
stats.changed_cards.append(key)
|
|
293
|
+
if prev is None:
|
|
294
|
+
stats.added += 1
|
|
295
|
+
else:
|
|
296
|
+
stats.updated += 1
|
|
297
|
+
if stats.changed:
|
|
298
|
+
self._resolve_all_links()
|
|
299
|
+
conn.execute("COMMIT")
|
|
300
|
+
except BaseException:
|
|
301
|
+
conn.execute("ROLLBACK")
|
|
302
|
+
raise
|
|
303
|
+
return stats
|
|
304
|
+
|
|
305
|
+
def rebuild(self, table: MountTable, exclude: Sequence[Path] = ()) -> SyncStats:
|
|
306
|
+
"""Drop all reconstructible state and re-index from disk with full sha256 verification."""
|
|
307
|
+
conn = self.conn
|
|
308
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
309
|
+
try:
|
|
310
|
+
self._reset_derived()
|
|
311
|
+
conn.execute("COMMIT")
|
|
312
|
+
except BaseException:
|
|
313
|
+
conn.execute("ROLLBACK")
|
|
314
|
+
raise
|
|
315
|
+
return self.sync(table, exclude, verify=True)
|
|
316
|
+
|
|
317
|
+
def _reset_derived(self) -> None:
|
|
318
|
+
"""Drop everything reconstructible from disk (telemetry and schema version survive)."""
|
|
319
|
+
for name in _DERIVED_TABLES:
|
|
320
|
+
self.conn.execute(f"DELETE FROM {name}")
|
|
321
|
+
self.conn.execute("DELETE FROM meta WHERE key <> 'schema_version'")
|
|
322
|
+
|
|
323
|
+
def _delete_mount_rows(self, alias: str) -> int:
|
|
324
|
+
ids = [r[0] for r in self.conn.execute("SELECT id FROM cards WHERE mount=?", (alias,))]
|
|
325
|
+
self.conn.executemany("DELETE FROM cards_fts WHERE rowid=?", [(i,) for i in ids])
|
|
326
|
+
for tbl in ("cards", "card_aliases", "card_tags"):
|
|
327
|
+
self.conn.execute(f"DELETE FROM {tbl} WHERE mount=?", (alias,))
|
|
328
|
+
self.conn.execute("DELETE FROM links WHERE src_mount=?", (alias,))
|
|
329
|
+
return len(ids)
|
|
330
|
+
|
|
331
|
+
def _delete_card(self, mount: str, path: str) -> None:
|
|
332
|
+
row = self.conn.execute("SELECT id FROM cards WHERE mount=? AND path=?", (mount, path)).fetchone()
|
|
333
|
+
if row is not None:
|
|
334
|
+
self.conn.execute("DELETE FROM cards_fts WHERE rowid=?", (row[0],))
|
|
335
|
+
self.conn.execute("DELETE FROM cards WHERE mount=? AND path=?", (mount, path))
|
|
336
|
+
self.conn.execute("DELETE FROM card_aliases WHERE mount=? AND path=?", (mount, path))
|
|
337
|
+
self.conn.execute("DELETE FROM card_tags WHERE mount=? AND path=?", (mount, path))
|
|
338
|
+
self.conn.execute("DELETE FROM links WHERE src_mount=? AND src_path=?", (mount, path))
|
|
339
|
+
|
|
340
|
+
def _upsert_card(self, mount: str, path: str, data: bytes, digest: str, mtime_ns: int, size: int) -> None:
|
|
341
|
+
self._delete_card(mount, path)
|
|
342
|
+
text = data.decode("utf-8", errors="replace")
|
|
343
|
+
parsed = parse_card(text)
|
|
344
|
+
title = parsed.display_title(path)
|
|
345
|
+
summary, body = parsed.index_parts() # plain Markdown: the lead paragraph is the summary
|
|
346
|
+
preview, _ = preview_text(summary, body, limit=10_000)
|
|
347
|
+
lines = text.count("\n") + (0 if text.endswith("\n") or not text else 1)
|
|
348
|
+
cur = self.conn.execute(
|
|
349
|
+
"""INSERT INTO cards(mount, path, title, aliases, tags, extends, supersedes, summary, preview,
|
|
350
|
+
sha256, mtime_ns, size, lines, is_overview, has_frontmatter, load, errors, warnings)
|
|
351
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
352
|
+
(
|
|
353
|
+
mount,
|
|
354
|
+
path,
|
|
355
|
+
title,
|
|
356
|
+
json.dumps(parsed.aliases, ensure_ascii=False),
|
|
357
|
+
json.dumps(parsed.tags, ensure_ascii=False),
|
|
358
|
+
json.dumps(parsed.extends, ensure_ascii=False),
|
|
359
|
+
json.dumps(parsed.supersedes, ensure_ascii=False),
|
|
360
|
+
summary,
|
|
361
|
+
preview,
|
|
362
|
+
digest,
|
|
363
|
+
mtime_ns,
|
|
364
|
+
size,
|
|
365
|
+
lines,
|
|
366
|
+
1 if path == OVERVIEW_PATH else 0,
|
|
367
|
+
FRONTMATTER_CODES.get(parsed.frontmatter_format, 0),
|
|
368
|
+
parsed.load,
|
|
369
|
+
json.dumps(parsed.errors, ensure_ascii=False),
|
|
370
|
+
json.dumps(parsed.warnings, ensure_ascii=False),
|
|
371
|
+
),
|
|
372
|
+
)
|
|
373
|
+
card_id = cur.lastrowid
|
|
374
|
+
alias_text = " ".join(parsed.aliases + [path.replace("/", " ")])
|
|
375
|
+
fields = (title, alias_text, " ".join(parsed.tags), summary, body)
|
|
376
|
+
self.conn.execute(
|
|
377
|
+
"INSERT INTO cards_fts(rowid, title, aliases, tags, summary, body) VALUES(?, ?, ?, ?, ?, ?)",
|
|
378
|
+
(card_id, *(index_text(t) for t in fields)),
|
|
379
|
+
)
|
|
380
|
+
self.conn.executemany(
|
|
381
|
+
"INSERT INTO card_aliases(mount, path, alias) VALUES(?, ?, ?)",
|
|
382
|
+
[(mount, path, k) for k in dict.fromkeys(fold(a) for a in parsed.aliases) if k],
|
|
383
|
+
)
|
|
384
|
+
self.conn.executemany(
|
|
385
|
+
"INSERT INTO card_tags(mount, path, tag) VALUES(?, ?, ?)",
|
|
386
|
+
[(mount, path, k) for k in dict.fromkeys(fold(t) for t in parsed.tags) if k],
|
|
387
|
+
)
|
|
388
|
+
refs: list[tuple[str, int, str]] = []
|
|
389
|
+
for kind, values in (("extends", parsed.extends), ("supersedes", parsed.supersedes)):
|
|
390
|
+
refs += [(kind, i, v) for i, v in enumerate(values)]
|
|
391
|
+
refs += [("wikilink", i, v) for i, v in enumerate(parsed.wikilink_targets())]
|
|
392
|
+
self.conn.executemany(
|
|
393
|
+
"INSERT INTO links(src_mount, src_path, kind, ord, raw) VALUES(?, ?, ?, ?, ?)",
|
|
394
|
+
[(mount, path, kind, i, raw) for kind, i, raw in refs],
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
def catalogs(self) -> dict[str, MountCatalog]:
|
|
398
|
+
cats: dict[str, MountCatalog] = {}
|
|
399
|
+
for row in self.conn.execute("SELECT alias FROM mounts"):
|
|
400
|
+
cats[row[0]] = MountCatalog()
|
|
401
|
+
for row in self.conn.execute("SELECT mount, path, aliases FROM cards ORDER BY mount, path"):
|
|
402
|
+
cats.setdefault(row[0], MountCatalog()).add(row[1], json.loads(row[2]))
|
|
403
|
+
return cats
|
|
404
|
+
|
|
405
|
+
def _resolve_all_links(self) -> None:
|
|
406
|
+
cats = self.catalogs()
|
|
407
|
+
updates = []
|
|
408
|
+
for row in self.conn.execute("SELECT id, src_mount, raw FROM links").fetchall():
|
|
409
|
+
res = resolve_ref(row["raw"], row["src_mount"], cats)
|
|
410
|
+
updates.append((res.mount, res.path, 1 if res.resolved else 0, row["id"]))
|
|
411
|
+
self.conn.executemany("UPDATE links SET dst_mount=?, dst_path=?, resolved=? WHERE id=?", updates)
|
|
412
|
+
|
|
413
|
+
# ------------------------------------------------------------------
|
|
414
|
+
# queries
|
|
415
|
+
# ------------------------------------------------------------------
|
|
416
|
+
def mount_rows(self) -> list[sqlite3.Row]:
|
|
417
|
+
return self.conn.execute("SELECT * FROM mounts ORDER BY precedence DESC, alias").fetchall()
|
|
418
|
+
|
|
419
|
+
def card(self, mount: str, path: str) -> sqlite3.Row | None:
|
|
420
|
+
return self.conn.execute(
|
|
421
|
+
"""SELECT c.*, m.precedence, m.role FROM cards c JOIN mounts m ON m.alias = c.mount
|
|
422
|
+
WHERE c.mount=? AND c.path=?""",
|
|
423
|
+
(mount, path),
|
|
424
|
+
).fetchone()
|
|
425
|
+
|
|
426
|
+
def cards_at_path(self, path: str) -> list[sqlite3.Row]:
|
|
427
|
+
return self.conn.execute(
|
|
428
|
+
"""SELECT c.*, m.precedence, m.role FROM cards c JOIN mounts m ON m.alias = c.mount
|
|
429
|
+
WHERE c.path=? ORDER BY m.precedence DESC, c.mount""",
|
|
430
|
+
(path,),
|
|
431
|
+
).fetchall()
|
|
432
|
+
|
|
433
|
+
def cards_with_alias(self, alias: str, mount: str | None = None) -> list[sqlite3.Row]:
|
|
434
|
+
sql = """SELECT c.*, m.precedence, m.role FROM card_aliases a
|
|
435
|
+
JOIN cards c ON c.mount = a.mount AND c.path = a.path
|
|
436
|
+
JOIN mounts m ON m.alias = c.mount WHERE a.alias=?"""
|
|
437
|
+
args: list[Any] = [fold(alias)]
|
|
438
|
+
if mount is not None:
|
|
439
|
+
sql += " AND a.mount=?"
|
|
440
|
+
args.append(mount)
|
|
441
|
+
sql += " ORDER BY m.precedence DESC, c.mount, c.path"
|
|
442
|
+
return self.conn.execute(sql, args).fetchall()
|
|
443
|
+
|
|
444
|
+
def always_cards(self) -> list[sqlite3.Row]:
|
|
445
|
+
"""Cards with ``load = "always"``, base layers first (precedence ascending), then mount, path."""
|
|
446
|
+
return self.conn.execute(
|
|
447
|
+
"""SELECT c.*, m.precedence, m.role FROM cards c JOIN mounts m ON m.alias = c.mount
|
|
448
|
+
WHERE c.load = 'always' ORDER BY m.precedence, c.mount, c.path"""
|
|
449
|
+
).fetchall()
|
|
450
|
+
|
|
451
|
+
def all_cards(self, mount: str | None = None) -> list[sqlite3.Row]:
|
|
452
|
+
sql = "SELECT c.*, m.precedence, m.role FROM cards c JOIN mounts m ON m.alias = c.mount"
|
|
453
|
+
args: tuple[Any, ...] = ()
|
|
454
|
+
if mount is not None:
|
|
455
|
+
sql += " WHERE c.mount=?"
|
|
456
|
+
args = (mount,)
|
|
457
|
+
return self.conn.execute(sql + " ORDER BY m.precedence DESC, c.mount, c.path", args).fetchall()
|
|
458
|
+
|
|
459
|
+
def search(
|
|
460
|
+
self, query: str, sources: Iterable[str] | None = None, tags: Iterable[str] | None = None
|
|
461
|
+
) -> list[Hit]:
|
|
462
|
+
"""All matches (overviews excluded), best first: ``score = -bm25(...)``, then precedence.
|
|
463
|
+
|
|
464
|
+
Each hit also carries ``coverage``: the IDF-weighted share of the query it matches (an
|
|
465
|
+
unspaced run earns the weighted share of its bigrams, or full credit when matched
|
|
466
|
+
verbatim). It is reported for callers and does not order results: on the retrieval
|
|
467
|
+
evaluation, coverage-first ordering lost to BM25 order (evals/retrieval/PLAN.md).
|
|
468
|
+
An empty query with ``tags`` lists that facet (score and coverage 0).
|
|
469
|
+
"""
|
|
470
|
+
plan = analyze_query(query)
|
|
471
|
+
where, args = self._scope(sources, tags)
|
|
472
|
+
if plan.units:
|
|
473
|
+
weights = ", ".join(str(w) for w in BM25_WEIGHTS)
|
|
474
|
+
sql = f"""SELECT c.*, m.precedence, m.role, -bm25(cards_fts, {weights}) AS score
|
|
475
|
+
FROM cards_fts JOIN cards c ON c.id = cards_fts.rowid
|
|
476
|
+
JOIN mounts m ON m.alias = c.mount"""
|
|
477
|
+
where.insert(0, "cards_fts MATCH ?")
|
|
478
|
+
args.insert(0, plan.match_expression) # its placeholder comes first, before the scope clauses
|
|
479
|
+
elif (query or "").strip():
|
|
480
|
+
return [] # punctuation-only query: nothing searchable
|
|
481
|
+
else:
|
|
482
|
+
sql = """SELECT c.*, m.precedence, m.role, 0.0 AS score FROM cards c
|
|
483
|
+
JOIN mounts m ON m.alias = c.mount"""
|
|
484
|
+
sql += " WHERE " + " AND ".join(where)
|
|
485
|
+
rows = self.conn.execute(sql, args).fetchall()
|
|
486
|
+
hits = self._rank(plan, rows)
|
|
487
|
+
hits.sort(key=Hit.rank_key)
|
|
488
|
+
return hits
|
|
489
|
+
|
|
490
|
+
@staticmethod
|
|
491
|
+
def _scope(sources: Iterable[str] | None, tags: Iterable[str] | None) -> tuple[list[str], list[Any]]:
|
|
492
|
+
"""WHERE clauses (over ``cards c``) and their arguments: the searchable cards in scope."""
|
|
493
|
+
where: list[str] = ["c.is_overview = 0"]
|
|
494
|
+
args: list[Any] = []
|
|
495
|
+
source_list = list(sources or [])
|
|
496
|
+
if source_list:
|
|
497
|
+
where.append(f"c.mount IN ({', '.join('?' * len(source_list))})")
|
|
498
|
+
args += source_list
|
|
499
|
+
tag_list = sorted({fold(t) for t in (tags or []) if fold(t)})
|
|
500
|
+
if tag_list:
|
|
501
|
+
where.append(
|
|
502
|
+
f"""(SELECT COUNT(DISTINCT t.tag) FROM card_tags t WHERE t.mount = c.mount AND t.path = c.path
|
|
503
|
+
AND t.tag IN ({', '.join('?' * len(tag_list))})) = ?"""
|
|
504
|
+
)
|
|
505
|
+
args += tag_list + [len(tag_list)]
|
|
506
|
+
return where, args
|
|
507
|
+
|
|
508
|
+
def unmatched_terms(self, query: str, hits: Sequence[Hit]) -> list[str]:
|
|
509
|
+
"""Query words (an unspaced run counts as one) that no card in scope contains.
|
|
510
|
+
|
|
511
|
+
``hits`` must be ``search()``'s result for the same query and scope: a word is unmatched
|
|
512
|
+
when none of its terms occurs in any hit. It is a plain fact about the corpus, with no
|
|
513
|
+
heuristics, in every script; it changes and reorders nothing. (Spelling suggestions were
|
|
514
|
+
tried and removed: evals/retrieval/PLAN.md, "Decision: no did_you_mean".)
|
|
515
|
+
"""
|
|
516
|
+
plan = analyze_query(query)
|
|
517
|
+
ids = {h.row["id"] for h in hits}
|
|
518
|
+
if not ids:
|
|
519
|
+
return [u.text for u in plan.units]
|
|
520
|
+
if len(plan.units) == 1:
|
|
521
|
+
return []
|
|
522
|
+
return [u.text for u in plan.units if not any(self._matching_ids(t) & ids for t in u.terms)]
|
|
523
|
+
|
|
524
|
+
def _matching_ids(self, expression: str) -> set[int]:
|
|
525
|
+
return {r[0] for r in self.conn.execute("SELECT rowid FROM cards_fts WHERE cards_fts MATCH ?", (expression,))}
|
|
526
|
+
|
|
527
|
+
def _rank(self, plan: QueryPlan, rows: list[sqlite3.Row]) -> list[Hit]:
|
|
528
|
+
if not plan.units:
|
|
529
|
+
return [Hit(r, 0.0, 0.0, 0) for r in rows]
|
|
530
|
+
if len(plan.units) == 1 and len(plan.units[0].terms) == 1:
|
|
531
|
+
return [Hit(r, float(r["score"]), 1.0, 0) for r in rows] # every match matches everything
|
|
532
|
+
term_ids = {t: self._matching_ids(t) for t in plan.terms}
|
|
533
|
+
phrase_ids = {p: self._matching_ids(p) for p in plan.phrases}
|
|
534
|
+
n_cards = self.conn.execute("SELECT COUNT(*) FROM cards").fetchone()[0]
|
|
535
|
+
# Smoothed BM25 IDF (always positive, also in tiny corpora where FTS5 clamps its own to ~0).
|
|
536
|
+
idf = {t: math.log(1 + (n_cards - len(ids) + 0.5) / (len(ids) + 0.5)) for t, ids in term_ids.items()}
|
|
537
|
+
unit_weight = [sum(idf[t] for t in u.terms) / len(u.terms) for u in plan.units]
|
|
538
|
+
total = sum(unit_weight)
|
|
539
|
+
hits = []
|
|
540
|
+
for r in rows:
|
|
541
|
+
rid = r["id"]
|
|
542
|
+
credit = 0.0
|
|
543
|
+
phrases = 0
|
|
544
|
+
for unit, weight in zip(plan.units, unit_weight):
|
|
545
|
+
if unit.phrase and rid in phrase_ids[unit.phrase]:
|
|
546
|
+
credit += weight
|
|
547
|
+
phrases += 1
|
|
548
|
+
continue
|
|
549
|
+
matched = sum(idf[t] for t in unit.terms if rid in term_ids[t])
|
|
550
|
+
credit += weight * matched / sum(idf[t] for t in unit.terms)
|
|
551
|
+
hits.append(Hit(r, float(r["score"]), credit / total if total else 0.0, phrases))
|
|
552
|
+
return hits
|
|
553
|
+
|
|
554
|
+
def card_has_tags(self, mount: str, path: str, tags: Iterable[str]) -> bool:
|
|
555
|
+
wanted = {fold(t) for t in tags if fold(t)}
|
|
556
|
+
if not wanted:
|
|
557
|
+
return True
|
|
558
|
+
have = {r[0] for r in self.conn.execute("SELECT tag FROM card_tags WHERE mount=? AND path=?", (mount, path))}
|
|
559
|
+
return wanted <= have
|
|
560
|
+
|
|
561
|
+
def inbound(self, kind: str, mount: str, path: str, recency: bool = False) -> list[sqlite3.Row]:
|
|
562
|
+
"""Cards whose ``kind`` reference resolves to ``(mount, path)``.
|
|
563
|
+
|
|
564
|
+
Ordered by precedence, then address (deterministic). ``recency=True`` orders by
|
|
565
|
+
precedence then newest first (the design's order for ``extensions``).
|
|
566
|
+
"""
|
|
567
|
+
order = "m.precedence DESC, c.mtime_ns DESC," if recency else "m.precedence DESC,"
|
|
568
|
+
return self.conn.execute(
|
|
569
|
+
"""SELECT DISTINCT l.src_mount, l.src_path, c.mtime_ns, m.precedence, m.role
|
|
570
|
+
FROM links l JOIN cards c ON c.mount = l.src_mount AND c.path = l.src_path
|
|
571
|
+
JOIN mounts m ON m.alias = l.src_mount
|
|
572
|
+
WHERE l.kind=? AND l.dst_mount=? AND l.dst_path=? AND l.resolved=1
|
|
573
|
+
AND NOT (l.src_mount = l.dst_mount AND l.src_path = l.dst_path)
|
|
574
|
+
ORDER BY """ + order + " l.src_mount, l.src_path",
|
|
575
|
+
(kind, mount, path),
|
|
576
|
+
).fetchall()
|
|
577
|
+
|
|
578
|
+
def dangling_inbound(self, kind: str, mount: str, path: str) -> list[sqlite3.Row]:
|
|
579
|
+
"""Cards whose ``kind`` reference names ``(mount, path)`` by path but no longer resolves (a deleted target)."""
|
|
580
|
+
return self.conn.execute(
|
|
581
|
+
"""SELECT DISTINCT l.src_mount, l.src_path, c.mtime_ns, m.precedence, m.role
|
|
582
|
+
FROM links l JOIN cards c ON c.mount = l.src_mount AND c.path = l.src_path
|
|
583
|
+
JOIN mounts m ON m.alias = l.src_mount
|
|
584
|
+
WHERE l.kind=? AND l.dst_mount=? AND l.dst_path=? AND l.resolved=0
|
|
585
|
+
ORDER BY m.precedence DESC, l.src_mount, l.src_path""",
|
|
586
|
+
(kind, mount, path),
|
|
587
|
+
).fetchall()
|
|
588
|
+
|
|
589
|
+
def outbound(self, mount: str, path: str, kind: str | None = None) -> list[sqlite3.Row]:
|
|
590
|
+
sql = "SELECT kind, ord, raw, dst_mount, dst_path, resolved FROM links WHERE src_mount=? AND src_path=?"
|
|
591
|
+
args: list[Any] = [mount, path]
|
|
592
|
+
if kind is not None:
|
|
593
|
+
sql += " AND kind=?"
|
|
594
|
+
args.append(kind)
|
|
595
|
+
return self.conn.execute(sql + " ORDER BY kind, ord", args).fetchall()
|
|
596
|
+
|
|
597
|
+
def lineage_edges(self) -> list[sqlite3.Row]:
|
|
598
|
+
return self.conn.execute(
|
|
599
|
+
"""SELECT src_mount, src_path, kind, raw, dst_mount, dst_path, resolved FROM links
|
|
600
|
+
WHERE kind IN ('extends', 'supersedes')"""
|
|
601
|
+
).fetchall()
|
|
602
|
+
|
|
603
|
+
def folder_counts(self, mount: str) -> tuple[list[tuple[str, int]], int, int]:
|
|
604
|
+
"""``([(folder, card_count)...], root_card_count, total_cards)`` for one mount (overview excluded)."""
|
|
605
|
+
folders: dict[str, int] = {}
|
|
606
|
+
root = 0
|
|
607
|
+
total = 0
|
|
608
|
+
for row in self.conn.execute("SELECT path FROM cards WHERE mount=? AND is_overview=0", (mount,)):
|
|
609
|
+
total += 1
|
|
610
|
+
head, sep, _ = row[0].partition("/")
|
|
611
|
+
if sep:
|
|
612
|
+
folders[head] = folders.get(head, 0) + 1
|
|
613
|
+
else:
|
|
614
|
+
root += 1
|
|
615
|
+
ordered = sorted(folders.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
616
|
+
return ordered, root, total
|
|
617
|
+
|
|
618
|
+
# ------------------------------------------------------------------
|
|
619
|
+
# telemetry (best effort)
|
|
620
|
+
# ------------------------------------------------------------------
|
|
621
|
+
def bump_read(self, mount: str, path: str) -> tuple[int, float] | None:
|
|
622
|
+
now = time.time()
|
|
623
|
+
try:
|
|
624
|
+
self.conn.execute(
|
|
625
|
+
"""INSERT INTO telemetry(mount, path, read_count, last_read_at) VALUES(?, ?, 1, ?)
|
|
626
|
+
ON CONFLICT(mount, path) DO UPDATE SET read_count = read_count + 1, last_read_at = excluded.last_read_at""",
|
|
627
|
+
(mount, path, now),
|
|
628
|
+
)
|
|
629
|
+
row = self.conn.execute(
|
|
630
|
+
"SELECT read_count, last_read_at FROM telemetry WHERE mount=? AND path=?", (mount, path)
|
|
631
|
+
).fetchone()
|
|
632
|
+
return (row[0], row[1]) if row else None
|
|
633
|
+
except sqlite3.Error:
|
|
634
|
+
return None
|