renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,670 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.runMigrations = runMigrations;
4
- const log_1 = require("../mvp/log");
5
- const log = (0, log_1.createLogger)('engine', { stderr: true });
6
- const migrations = [
7
- {
8
- version: 1,
9
- name: 'initial_schema',
10
- up: (db) => {
11
- const ddl = [
12
- `CREATE TABLE IF NOT EXISTS notes (
13
- path TEXT PRIMARY KEY,
14
- title TEXT NOT NULL,
15
- kind TEXT,
16
- note_hash TEXT NOT NULL,
17
- modified_at TEXT NOT NULL,
18
- frontmatter_json TEXT NOT NULL DEFAULT '{}'
19
- )`,
20
- `CREATE TABLE IF NOT EXISTS chunks (
21
- id INTEGER PRIMARY KEY AUTOINCREMENT,
22
- note_path TEXT NOT NULL REFERENCES notes(path) ON DELETE CASCADE,
23
- heading_path TEXT NOT NULL DEFAULT '',
24
- text TEXT NOT NULL,
25
- start_line INTEGER NOT NULL,
26
- end_line INTEGER NOT NULL,
27
- token_count INTEGER NOT NULL,
28
- chunk_hash TEXT NOT NULL
29
- )`,
30
- `CREATE TABLE IF NOT EXISTS entities (
31
- id INTEGER PRIMARY KEY AUTOINCREMENT,
32
- type TEXT NOT NULL,
33
- canonical_name TEXT NOT NULL,
34
- aliases_json TEXT NOT NULL DEFAULT '[]',
35
- source_note TEXT NOT NULL REFERENCES notes(path) ON DELETE CASCADE,
36
- confidence REAL NOT NULL DEFAULT 1.0,
37
- updated_at TEXT NOT NULL
38
- )`,
39
- `CREATE TABLE IF NOT EXISTS facts (
40
- id INTEGER PRIMARY KEY AUTOINCREMENT,
41
- subject_entity_id INTEGER REFERENCES entities(id) ON DELETE CASCADE,
42
- predicate TEXT NOT NULL,
43
- object_text TEXT,
44
- object_entity_id INTEGER REFERENCES entities(id) ON DELETE SET NULL,
45
- source_path TEXT NOT NULL,
46
- source_chunk_id INTEGER REFERENCES chunks(id) ON DELETE SET NULL,
47
- confidence REAL NOT NULL DEFAULT 1.0,
48
- valid_from TEXT NOT NULL,
49
- valid_to TEXT,
50
- updated_at TEXT NOT NULL
51
- )`,
52
- `CREATE TABLE IF NOT EXISTS relations (
53
- id INTEGER PRIMARY KEY AUTOINCREMENT,
54
- source_entity_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
55
- relation TEXT NOT NULL,
56
- target_entity_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
57
- source_note TEXT NOT NULL,
58
- confidence REAL NOT NULL DEFAULT 1.0
59
- )`,
60
- `CREATE TABLE IF NOT EXISTS write_events (
61
- id INTEGER PRIMARY KEY AUTOINCREMENT,
62
- note_path TEXT NOT NULL,
63
- action TEXT NOT NULL,
64
- summary TEXT,
65
- created_at TEXT NOT NULL
66
- )`,
67
- `CREATE INDEX IF NOT EXISTS idx_chunks_note ON chunks(note_path)`,
68
- `CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(canonical_name)`,
69
- `CREATE INDEX IF NOT EXISTS idx_facts_source ON facts(source_path)`,
70
- `CREATE INDEX IF NOT EXISTS idx_relations_source ON relations(source_entity_id)`,
71
- `CREATE INDEX IF NOT EXISTS idx_relations_target ON relations(target_entity_id)`,
72
- `CREATE INDEX IF NOT EXISTS idx_entities_source_note ON entities(source_note)`,
73
- `CREATE INDEX IF NOT EXISTS idx_facts_subject ON facts(subject_entity_id)`,
74
- ];
75
- for (const sql of ddl)
76
- db.prepare(sql).run();
77
- },
78
- },
79
- {
80
- version: 2,
81
- name: 'uniqueness_constraints',
82
- up: (db) => {
83
- db.prepare(`CREATE UNIQUE INDEX IF NOT EXISTS idx_relations_unique ON relations(source_entity_id, relation, target_entity_id)`).run();
84
- db.prepare(`CREATE UNIQUE INDEX IF NOT EXISTS idx_facts_unique ON facts(subject_entity_id, predicate, object_text)`).run();
85
- },
86
- },
87
- {
88
- version: 3,
89
- name: 'fts5_chunks',
90
- up: () => {
91
- // Intentionally left as no-op — superseded by migration 4
92
- },
93
- },
94
- {
95
- version: 4,
96
- name: 'fts5_chunks_fixed',
97
- up: (db) => {
98
- // Drop any partial v3 artefacts
99
- db.prepare(`DROP TABLE IF EXISTS chunks_fts`).run();
100
- db.prepare(`DROP TRIGGER IF EXISTS chunks_fts_insert`).run();
101
- db.prepare(`DROP TRIGGER IF EXISTS chunks_fts_delete`).run();
102
- db.prepare(`DROP TRIGGER IF EXISTS chunks_fts_update`).run();
103
- db.prepare(`
104
- CREATE VIRTUAL TABLE chunks_fts
105
- USING fts5(text, heading_path, note_path UNINDEXED, content='chunks', content_rowid='id')
106
- `).run();
107
- // Populate from existing chunks
108
- db.prepare(`INSERT INTO chunks_fts(rowid, text, heading_path, note_path) SELECT id, text, heading_path, note_path FROM chunks`).run();
109
- // Triggers to keep FTS in sync with chunks table
110
- db.prepare(`
111
- CREATE TRIGGER chunks_fts_insert AFTER INSERT ON chunks BEGIN
112
- INSERT INTO chunks_fts(rowid, text, heading_path, note_path) VALUES (new.id, new.text, new.heading_path, new.note_path);
113
- END
114
- `).run();
115
- db.prepare(`
116
- CREATE TRIGGER chunks_fts_delete AFTER DELETE ON chunks BEGIN
117
- INSERT INTO chunks_fts(chunks_fts, rowid, text, heading_path, note_path) VALUES ('delete', old.id, old.text, old.heading_path, old.note_path);
118
- END
119
- `).run();
120
- db.prepare(`
121
- CREATE TRIGGER chunks_fts_update AFTER UPDATE ON chunks BEGIN
122
- INSERT INTO chunks_fts(chunks_fts, rowid, text, heading_path, note_path) VALUES ('delete', old.id, old.text, old.heading_path, old.note_path);
123
- INSERT INTO chunks_fts(rowid, text, heading_path, note_path) VALUES (new.id, new.text, new.heading_path, new.note_path);
124
- END
125
- `).run();
126
- },
127
- },
128
- {
129
- version: 5,
130
- name: '5_facts_event_id',
131
- up: (db) => {
132
- const cols = db.prepare('PRAGMA table_info(facts)').all().map((c) => c.name);
133
- if (!cols.includes('event_id')) {
134
- db.exec('ALTER TABLE facts ADD COLUMN event_id TEXT');
135
- db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_facts_event ON facts(event_id)');
136
- }
137
- },
138
- },
139
- {
140
- version: 6,
141
- name: '6_lifecycle_event_identity',
142
- up: (db) => {
143
- // B1: uniqueness is event identity, not (subject, predicate, object).
144
- // The v2 index made A→B→A supersession impossible (replay and even the
145
- // live write threw UNIQUE) — closed history may legitimately repeat a
146
- // value. Replace it with the engine's real invariant: at most ONE
147
- // current (valid_to IS NULL) fact per (subject_entity_id, predicate).
148
- db.prepare('DROP INDEX IF EXISTS idx_facts_unique').run();
149
- const now = new Date().toISOString();
150
- // Repair pre-existing violations of that invariant — SCOPED to canonical
151
- // lifecycle facts (source_path '_mcp_'): only lifecycle streams are
152
- // single-current by design. Extracted/frontmatter facts may legitimately
153
- // hold multiple current values per (entity, predicate) and are NOT
154
- // touched. Deterministically keep the newest current fact per
155
- // (subject, predicate) and close the rest. NOTE (K12): this interim
156
- // choice is made without ledger access (migrations are DB-only); the
157
- // startup replay that follows canonicalizes every lifecycle stream from
158
- // ledger order (reconcileStreamCurrent) BEFORE readiness, so any
159
- // non-canonical survivor/valid_to this repair leaves is corrected there.
160
- db.prepare(`
161
- UPDATE facts SET valid_to = ?, updated_at = ?
162
- WHERE source_path = '_mcp_' AND id IN (
163
- SELECT id FROM (
164
- SELECT f.id AS id,
165
- ROW_NUMBER() OVER (
166
- PARTITION BY f.subject_entity_id, f.predicate
167
- ORDER BY f.valid_from DESC, f.id DESC
168
- ) AS rn
169
- FROM facts f WHERE f.valid_to IS NULL AND f.source_path = '_mcp_'
170
- ) WHERE rn > 1
171
- )
172
- `).run(now, now);
173
- db.prepare(`CREATE UNIQUE INDEX IF NOT EXISTS idx_facts_current ON facts(subject_entity_id, predicate) WHERE valid_to IS NULL AND source_path = '_mcp_'`).run();
174
- // B1 replay idempotency: byte-identical events dedup, a reused event id
175
- // with a conflicting payload fails explicitly instead of being skipped.
176
- db.exec(`
177
- CREATE TABLE IF NOT EXISTS applied_ledger_events (
178
- event_id TEXT PRIMARY KEY,
179
- digest TEXT NOT NULL,
180
- applied_at TEXT NOT NULL
181
- )
182
- `);
183
- },
184
- },
185
- {
186
- version: 7,
187
- name: '7_scope_lifecycle_current_index',
188
- up: (db) => {
189
- // Repair pass (Astra A1/Kimi K3): the first Gate1A candidate's migration 6
190
- // scoped the single-current index to ALL facts, which silently rejected
191
- // legitimate extracted/frontmatter facts sharing (entity, predicate) and
192
- // (in its duplicate repair) closed unrelated current rows. Re-scope the
193
- // index to canonical lifecycle facts only (source_path '_mcp_'). This
194
- // also upgrades DBs that already applied the candidate's globally-scoped
195
- // migration 6. Limitation (documented in GATE1A-EVIDENCE.md): duplicate
196
- // non-lifecycle current rows that the candidate's global v6 repair
197
- // already closed cannot be un-closed; recovery is a rebuild/reindex,
198
- // which re-extracts them from the source notes.
199
- db.prepare('DROP INDEX IF EXISTS idx_facts_current').run();
200
- db.prepare(`CREATE UNIQUE INDEX IF NOT EXISTS idx_facts_current ON facts(subject_entity_id, predicate) WHERE valid_to IS NULL AND source_path = '_mcp_'`).run();
201
- },
202
- },
203
- {
204
- version: 8,
205
- name: '8_facts_scope',
206
- up: (db) => {
207
- // Optional product scope on lifecycle facts. Missing/legacy = 'global'.
208
- // Uniqueness becomes (subject_entity_id, predicate, scope) for current
209
- // _mcp_ facts so the same subject+predicate can coexist across scopes.
210
- const cols = db.prepare('PRAGMA table_info(facts)').all().map((c) => c.name);
211
- if (!cols.includes('scope')) {
212
- db.exec(`ALTER TABLE facts ADD COLUMN scope TEXT NOT NULL DEFAULT 'global'`);
213
- }
214
- db.prepare('DROP INDEX IF EXISTS idx_facts_current').run();
215
- db.prepare(`CREATE UNIQUE INDEX IF NOT EXISTS idx_facts_current ON facts(subject_entity_id, predicate, scope) WHERE valid_to IS NULL AND source_path = '_mcp_'`).run();
216
- },
217
- },
218
- {
219
- version: 9,
220
- name: '9_structured_rows',
221
- up: (db) => {
222
- // C0 (ADR-0003 §4): one provenance-backed structured-row store for the
223
- // fact/event/preference extraction lanes. Every row traces to its source
224
- // note via source_path FK ON DELETE CASCADE — the legacy `facts` table
225
- // (v1) has no FK and the watcher's unlink handler orphans rows today
226
- // (src/watcher/watcher.ts), so this migration also performs a one-time
227
- // purge of already-orphaned legacy fact rows.
228
- db.exec(`
229
- CREATE TABLE IF NOT EXISTS structured_rows (
230
- id INTEGER PRIMARY KEY AUTOINCREMENT,
231
- row_kind TEXT NOT NULL CHECK (row_kind IN ('fact', 'event', 'preference')),
232
- source_path TEXT NOT NULL REFERENCES notes(path) ON DELETE CASCADE,
233
- source_hash TEXT NOT NULL,
234
- source_chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
235
- scope TEXT NOT NULL DEFAULT 'global',
236
- extraction_version TEXT NOT NULL,
237
- valid_from TEXT,
238
- valid_to TEXT,
239
- payload_json TEXT NOT NULL,
240
- dedupe_key TEXT NOT NULL,
241
- created_at TEXT NOT NULL
242
- )
243
- `);
244
- db.exec(`CREATE INDEX IF NOT EXISTS idx_structured_rows_source ON structured_rows(source_path)`);
245
- db.exec(`CREATE INDEX IF NOT EXISTS idx_structured_rows_current ON structured_rows(row_kind, scope, dedupe_key) WHERE valid_to IS NULL`);
246
- // One-time orphan purge: rows in the legacy (FK-less) `facts` table whose
247
- // source note no longer resolves can never be composed (no provenance);
248
- // remove them instead of carrying a permanent live defect.
249
- const hasFacts = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='facts'").get();
250
- if (hasFacts) {
251
- db.prepare('DELETE FROM facts WHERE source_path NOT IN (SELECT path FROM notes)').run();
252
- }
253
- },
254
- },
255
- {
256
- version: 10,
257
- name: '10_structured_rows_confirm_count',
258
- up: (db) => {
259
- // C2 facts write-path: confirm count for dedupe adjudication (HYP card
260
- // plan step 2 — dedupe on (value, NFKD-ASCII-lowercased object) with a
261
- // confirm count). Identical re-extraction bumps the count instead of
262
- // writing a duplicate row.
263
- const cols = db.prepare('PRAGMA table_info(structured_rows)').all().map((c) => c.name);
264
- if (!cols.includes('confirm_count')) {
265
- db.exec('ALTER TABLE structured_rows ADD COLUMN confirm_count INTEGER NOT NULL DEFAULT 1');
266
- }
267
- },
268
- },
269
- {
270
- version: 11,
271
- name: '11_structured_rows_current_identity_unique',
272
- up: (db) => {
273
- // C2 review fix (P2): close the check-then-act race across `await
274
- // model.decide` — concurrent same-identity inserts from the same source
275
- // note can no longer double-insert. Partial index: only current rows
276
- // (valid_to IS NULL) participate, so closed/superseded history may
277
- // legitimately repeat an identity, and the source_path column keeps
278
- // cross-note NULL-date coexistence intact. Pre-repair: deterministically
279
- // close any pre-existing same-source duplicate current rows (keep the
280
- // lowest id — the same row the dedupe adjudication reads first), so the
281
- // index creation cannot fail on legacy data.
282
- const now = new Date().toISOString();
283
- db.prepare(`
284
- UPDATE structured_rows SET valid_to = ?
285
- WHERE valid_to IS NULL AND id IN (
286
- SELECT id FROM (
287
- SELECT r.id AS id,
288
- ROW_NUMBER() OVER (
289
- PARTITION BY r.row_kind, r.scope, r.dedupe_key, r.source_path
290
- ORDER BY r.id
291
- ) AS rn
292
- FROM structured_rows r WHERE r.valid_to IS NULL
293
- ) WHERE rn > 1
294
- )
295
- `).run(now);
296
- db.prepare(`
297
- CREATE UNIQUE INDEX IF NOT EXISTS idx_structured_rows_current_identity
298
- ON structured_rows(row_kind, scope, dedupe_key, source_path)
299
- WHERE valid_to IS NULL
300
- `).run();
301
- },
302
- },
303
- {
304
- version: 12,
305
- name: '12_lane_runs',
306
- up: (db) => {
307
- // #189: the zero-yield half of the ADR-0003 §4.2 lane cache.
308
- //
309
- // The row-based half lives in structured_rows: a CURRENT row for
310
- // (source_path, row_kind, source_hash, extraction_version) means "this
311
- // lane is done for these bytes". A lane that RUNS and produces nothing
312
- // leaves no row, so that half cannot record it and every later pass over
313
- // the same unchanged note calls the extractor again — repeated model
314
- // calls and rate-limit exposure for a note that yielded nothing.
315
- //
316
- // This table is the checkpoint of the last COMPLETED run of one lane on
317
- // one note. It is deliberately NOT a row in structured_rows: that table
318
- // is the scored side of the eval boundary, and a marker row there would
319
- // be served as an extracted fact.
320
- //
321
- // `max_rows_written` is what a COMPLETED run's adjudicated writes
322
- // returned, kept as a MAXIMUM over the runs of one key. It is deliberately
323
- // NOT a count read back out of structured_rows: that was a snapshot of
324
- // mutable DB state, and a delayed writer could overwrite a truthful
325
- // nonzero record with a stale zero (primary review B1 at head aeb388e),
326
- // suppressing the lane for bytes that do yield rows. It is also
327
- // deliberately monotone (never lowered): two concurrent runs can disagree
328
- // about the same bytes — the extractor is a model — and the later-finishing
329
- // one must not be able to erase a positive outcome with a zero (review B1
330
- // at head 9397d50). 0 therefore means "no completed run for these bytes
331
- // ever wrote a row", which is the safe direction to skip on.
332
- //
333
- // Key shape is the same cache key plus the lane, so a zero-yield fact
334
- // lane can never suppress events or preferences on the same bytes.
335
- // Growth is bounded by the armed lanes being default-off and by the
336
- // indexer only touching changed notes; one row per (note, lane,
337
- // content-version) is the ceiling.
338
- db.exec(`
339
- CREATE TABLE IF NOT EXISTS lane_runs (
340
- source_path TEXT NOT NULL REFERENCES notes(path) ON DELETE CASCADE,
341
- row_kind TEXT NOT NULL CHECK (row_kind IN ('fact', 'event', 'preference')),
342
- source_hash TEXT NOT NULL,
343
- extraction_version TEXT NOT NULL,
344
- max_rows_written INTEGER NOT NULL,
345
- completed_at TEXT NOT NULL,
346
- PRIMARY KEY (source_path, row_kind, source_hash, extraction_version)
347
- )
348
- `);
349
- },
350
- },
351
- {
352
- version: 13,
353
- name: '13_note_identity_fts',
354
- up: (db) => {
355
- // An identifier that NAMES a note (slug, file name, frontmatter id) is not in
356
- // chunk text, so chunks_fts cannot find it. Index it once, per note, with `-_.%`
357
- // as token characters: `space-bunny-eval-1001` is then a single whole token and
358
- // a query for it matches exactly that token — not `4242`, not `md`.
359
- db.exec(`
360
- CREATE VIRTUAL TABLE IF NOT EXISTS note_identity_fts
361
- USING fts5(ident, tokenize = "unicode61 tokenchars '-_.%'")
362
- `);
363
- const ident = (n) => `(
364
- ${n}.path || ' ' || CASE WHEN ${n}.path LIKE '%.md' THEN substr(${n}.path, 1, length(${n}.path) - 3) ELSE ${n}.path END
365
- || ' ' || CASE WHEN json_valid(${n}.frontmatter_json)
366
- THEN coalesce((SELECT group_concat(value, ' ') FROM json_tree(${n}.frontmatter_json) WHERE type = 'text'), '')
367
- ELSE '' END
368
- )`;
369
- db.exec(`
370
- DROP TRIGGER IF EXISTS note_identity_fts_insert;
371
- DROP TRIGGER IF EXISTS note_identity_fts_update;
372
- DROP TRIGGER IF EXISTS note_identity_fts_delete;
373
- CREATE TRIGGER note_identity_fts_insert AFTER INSERT ON notes BEGIN
374
- INSERT INTO note_identity_fts(rowid, ident) VALUES (new.rowid, ${ident('new')});
375
- END;
376
- CREATE TRIGGER note_identity_fts_update AFTER UPDATE OF path, frontmatter_json ON notes BEGIN
377
- DELETE FROM note_identity_fts WHERE rowid = old.rowid;
378
- INSERT INTO note_identity_fts(rowid, ident) VALUES (new.rowid, ${ident('new')});
379
- END;
380
- CREATE TRIGGER note_identity_fts_delete AFTER DELETE ON notes BEGIN
381
- DELETE FROM note_identity_fts WHERE rowid = old.rowid;
382
- END;
383
- `);
384
- db.exec(`DELETE FROM note_identity_fts`);
385
- db.exec(`INSERT INTO note_identity_fts(rowid, ident) SELECT n.rowid, ${ident('n')} FROM notes n`);
386
- },
387
- },
388
- {
389
- version: 14,
390
- name: '14_fact_terms',
391
- up: (db) => {
392
- // Lifecycle facts are found by the words in them. This inverted index (memory/factTerms.ts)
393
- // lets a search read the rows of the query's words instead of every fact. The write path fills
394
- // it in the transaction that writes a fact and ensureFactTerms fills it for facts that already
395
- // exist, so the migration only creates the schema.
396
- db.exec(`
397
- CREATE TABLE IF NOT EXISTS fact_terms (
398
- term TEXT NOT NULL,
399
- scope TEXT NOT NULL,
400
- fact_id INTEGER NOT NULL,
401
- PRIMARY KEY (term, scope, fact_id)
402
- ) WITHOUT ROWID
403
- `);
404
- // The current lifecycle facts in time order (every scope, and one scope), for a read of the newest
405
- // with no query: read backwards, they need no sort. Ascending, so a new fact appends to them.
406
- // Partial, so note-derived facts never touch them.
407
- db.exec(`CREATE INDEX IF NOT EXISTS idx_facts_lifecycle_recent ON facts(valid_from, id) WHERE valid_to IS NULL AND source_path = '_mcp_'`);
408
- db.exec(`CREATE INDEX IF NOT EXISTS idx_facts_lifecycle_scope_recent ON facts(scope, valid_from, id) WHERE valid_to IS NULL AND source_path = '_mcp_'`);
409
- },
410
- },
411
- {
412
- version: 15,
413
- name: '15_notes_unfinished',
414
- up: (db) => {
415
- // A file whose last index pass did not finish (its embedding failed) keeps an empty note hash until a later
416
- // pass finishes it (indexer/pipeline.ts). A search asks whether there is one (mvp/engine.ts, UNFINISHED_FILES):
417
- // partial, so the answer comes from those few rows, usually none, not from every note.
418
- db.exec(`CREATE INDEX IF NOT EXISTS idx_notes_unfinished ON notes(path) WHERE note_hash = ''`);
419
- },
420
- },
421
- {
422
- version: 16,
423
- name: '16_foreign_key_indexes',
424
- up: (db) => {
425
- // Every index pass replaces a note's entity, and a pass over an edited or appended note deletes the chunks it
426
- // replaces. Each deleted row makes SQLite find the rows whose foreign keys point at it, and these three columns
427
- // had no index, so every save read every fact (and every structured row) once per row it deleted. A table
428
- // older than these columns has no such key to look up, and gets no index.
429
- const has = (table, column) => db.prepare(`PRAGMA table_info(${table})`).all().some(c => c.name === column);
430
- if (has('facts', 'source_chunk_id'))
431
- db.exec(`CREATE INDEX IF NOT EXISTS idx_facts_source_chunk ON facts(source_chunk_id)`);
432
- if (has('facts', 'object_entity_id'))
433
- db.exec(`CREATE INDEX IF NOT EXISTS idx_facts_object_entity ON facts(object_entity_id)`);
434
- if (has('structured_rows', 'source_chunk_id')) {
435
- db.exec(`CREATE INDEX IF NOT EXISTS idx_structured_rows_source_chunk ON structured_rows(source_chunk_id)`);
436
- }
437
- },
438
- },
439
- {
440
- version: 17,
441
- name: '17_retired_facts',
442
- up: (db) => {
443
- // A retired lifecycle fact (memory/lifecycle.ts, retireFact) leaves `facts` for this table, row and id unchanged,
444
- // so no read of `facts` can return it and restore puts back the same row. Like `facts`, it is derived from the
445
- // ledger's retire and restore events and rebuilt by a replay. `fact_retirements` lists those events, whose
446
- // ledger text a search never shows (retrieval/ledgerLiveness.ts).
447
- db.exec(`
448
- CREATE TABLE IF NOT EXISTS retired_facts (
449
- id INTEGER PRIMARY KEY,
450
- subject_entity_id INTEGER NOT NULL,
451
- predicate TEXT NOT NULL,
452
- object_text TEXT,
453
- confidence REAL NOT NULL,
454
- valid_from TEXT NOT NULL,
455
- valid_to TEXT,
456
- updated_at TEXT NOT NULL,
457
- event_id TEXT NOT NULL UNIQUE,
458
- scope TEXT NOT NULL,
459
- retire_event_id TEXT NOT NULL,
460
- retired_at TEXT NOT NULL,
461
- retired_by TEXT NOT NULL
462
- )
463
- `);
464
- db.exec(`CREATE INDEX IF NOT EXISTS idx_retired_facts_stream ON retired_facts(subject_entity_id, predicate)`);
465
- db.exec(`
466
- CREATE TABLE IF NOT EXISTS fact_retirements (
467
- event_id TEXT PRIMARY KEY,
468
- kind TEXT NOT NULL,
469
- target TEXT NOT NULL,
470
- at TEXT NOT NULL,
471
- source TEXT NOT NULL
472
- )
473
- `);
474
- },
475
- },
476
- {
477
- version: 18,
478
- name: '18_fact_author',
479
- up: (db) => {
480
- // Who wrote a lifecycle fact and a write event. Null on rows saved before a connection key had a name.
481
- // A table this database does not have (a partial legacy schema) is left alone.
482
- const add = (table, column) => {
483
- const cols = db.prepare(`PRAGMA table_info(${table})`).all();
484
- if (cols.length > 0 && !cols.some(c => c.name === column))
485
- db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} TEXT`);
486
- };
487
- add('facts', 'author');
488
- add('retired_facts', 'author');
489
- add('write_events', 'author');
490
- },
491
- },
492
- {
493
- version: 19,
494
- name: '19_whats_new_cursor',
495
- up: (db) => {
496
- const cols = db.prepare('PRAGMA table_info(write_events)').all();
497
- if (cols.length > 0 && !cols.some(c => c.name === 'scope'))
498
- db.exec('ALTER TABLE write_events ADD COLUMN scope TEXT');
499
- db.exec(`CREATE TABLE IF NOT EXISTS agent_cursors (
500
- agent TEXT NOT NULL,
501
- scope TEXT NOT NULL,
502
- last_at TEXT NOT NULL,
503
- last_id INTEGER NOT NULL,
504
- PRIMARY KEY (agent, scope)
505
- )`);
506
- },
507
- },
508
- {
509
- version: 20,
510
- name: '20_agent_requests',
511
- up: (db) => {
512
- // A request is a record. Nothing here runs a tool or writes a memory.
513
- db.exec(`CREATE TABLE IF NOT EXISTS agent_requests (
514
- id TEXT PRIMARY KEY,
515
- scope TEXT,
516
- from_agent TEXT NOT NULL,
517
- to_agent TEXT NOT NULL,
518
- question TEXT NOT NULL,
519
- answer TEXT,
520
- status TEXT NOT NULL CHECK (status IN ('open', 'answered', 'closed')),
521
- created_at TEXT NOT NULL,
522
- answered_at TEXT,
523
- closed_at TEXT
524
- )`);
525
- },
526
- },
527
- {
528
- version: 21,
529
- name: '21_known_agents',
530
- up: (db) => {
531
- // The connection names (x-renor-agent, set by the gateway from each key's label) this workspace has seen:
532
- // who ask_agent can address. ask_agent's hourly limit counts agent_requests by sender.
533
- db.exec(`CREATE TABLE IF NOT EXISTS known_agents (
534
- agent TEXT PRIMARY KEY,
535
- first_seen TEXT NOT NULL
536
- )`);
537
- db.exec('CREATE INDEX IF NOT EXISTS agent_requests_from_at ON agent_requests (from_agent, created_at)');
538
- },
539
- },
540
- {
541
- version: 22,
542
- name: '22_agent_leases',
543
- up: (db) => {
544
- // One live stdio process per hive name. The holder is pid:nonce; a dead pid may be replaced.
545
- db.exec(`CREATE TABLE IF NOT EXISTS agent_leases (
546
- agent TEXT PRIMARY KEY,
547
- holder TEXT NOT NULL,
548
- seen_at TEXT NOT NULL
549
- )`);
550
- },
551
- },
552
- {
553
- version: 23,
554
- name: '23_index_state',
555
- up: (db) => {
556
- // Small facts about the index itself that must survive a crash: `reindex_pending` is set when the rows of a moved
557
- // folder were removed, and cleared only once the folder is indexed again (see outsideRoot.ts).
558
- db.exec(`CREATE TABLE IF NOT EXISTS index_state (
559
- key TEXT PRIMARY KEY,
560
- value TEXT NOT NULL
561
- )`);
562
- },
563
- },
564
- {
565
- // Versions 22 (agent_leases) and 23 (index_state) are taken on main; this clock migration follows them.
566
- version: 24,
567
- name: '24_note_index_clock',
568
- up: (db) => {
569
- // Observation time, independent of a note's content-date window and of rows indexing can replace/delete.
570
- db.exec(`CREATE TABLE IF NOT EXISTS note_index_clock (
571
- scope TEXT PRIMARY KEY,
572
- observed_at TEXT NOT NULL
573
- )`);
574
- // Existing indexes already carry the indexer's own stamp in updated_at. Never seed from valid_from:
575
- // that can be a content date in the future. Scope spellings stay compatible with savedScopeSqlPredicate.
576
- db.exec(`INSERT INTO note_index_clock (scope, observed_at)
577
- SELECT COALESCE(scope, 'global'), MAX(updated_at) FROM facts WHERE source_path <> '_mcp_'
578
- GROUP BY COALESCE(scope, 'global')
579
- ON CONFLICT(scope) DO UPDATE SET observed_at = MAX(note_index_clock.observed_at, excluded.observed_at)`);
580
- },
581
- },
582
- {
583
- version: 25,
584
- name: '25_fact_statement_words',
585
- up: (db) => {
586
- // The words of the statement a lifecycle fact was saved with (memory/factTerms.ts `statementTerms`), so the facts
587
- // lane can count them toward a question's words. Null on a fact with no statement (save_memory) and on rows saved
588
- // before this column; the replay backfills those from the ledger (lifecycle.ts `backfillStatementWords`).
589
- // A table this database does not have (a partial legacy schema) is left alone.
590
- const add = (table, column) => {
591
- const cols = db.prepare(`PRAGMA table_info(${table})`).all();
592
- if (cols.length > 0 && !cols.some(c => c.name === column))
593
- db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} TEXT`);
594
- };
595
- add('facts', 'statement_words');
596
- add('retired_facts', 'statement_words');
597
- },
598
- },
599
- {
600
- version: 26,
601
- name: '26_fact_writers',
602
- up: (db) => {
603
- // Who saved each lifecycle fact (memory/writerConflicts.ts): the writer its connection authenticated, and the values
604
- // it replaced that their writers have not confirmed yet. Keyed by the fact's event, so retiring and restoring a
605
- // fact (which keep the event) keep its row, and a replay of the ledger writes it again. A fact with no row was saved
606
- // before this table: its writer is its source and it has no open displacements.
607
- db.exec(`
608
- CREATE TABLE IF NOT EXISTS fact_writers (
609
- event_id TEXT PRIMARY KEY,
610
- writer TEXT NOT NULL,
611
- displaced TEXT NOT NULL DEFAULT '[]'
612
- )
613
- `);
614
- },
615
- },
616
- {
617
- version: 27,
618
- name: '27_erased_facts',
619
- up: (db) => {
620
- // Content-free receipts for fact erasures. The authoritative tombstone is in the ledger; this projection makes
621
- // retries and restore refusals cheap without retaining subject, predicate, object, statement, scope, or source.
622
- db.exec(`CREATE TABLE IF NOT EXISTS erased_facts (
623
- event_id TEXT PRIMARY KEY,
624
- original_digest TEXT NOT NULL,
625
- erased_at TEXT NOT NULL
626
- )`);
627
- },
628
- },
629
- {
630
- version: 28,
631
- name: '28_erased_facts_closes',
632
- up: (db) => {
633
- // The closing link of an erasure (event ids and a time, no content), kept here as well as in the tombstone: a retry that
634
- // has to redo a lost rewrite cannot read it from the facts tables, which a restart may have reconciled without it.
635
- const cols = db.prepare('PRAGMA table_info(erased_facts)').all();
636
- if (!cols.some(c => c.name === 'closes'))
637
- db.exec('ALTER TABLE erased_facts ADD COLUMN closes TEXT');
638
- if (!cols.some(c => c.name === 'closed_at'))
639
- db.exec('ALTER TABLE erased_facts ADD COLUMN closed_at TEXT');
640
- },
641
- },
642
- ];
643
- function runMigrations(db) {
644
- db.prepare(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`).run();
645
- const row = db.prepare('SELECT version FROM schema_version LIMIT 1').get();
646
- let currentVersion = row?.version ?? 0;
647
- // If DB has no version but has tables (existing install), assume v1
648
- if (currentVersion === 0) {
649
- const hasNotes = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'").get();
650
- if (hasNotes) {
651
- migrations[0].up(db);
652
- currentVersion = 1;
653
- }
654
- }
655
- for (const migration of migrations) {
656
- if (migration.version <= currentVersion)
657
- continue;
658
- db.transaction(() => {
659
- migration.up(db);
660
- if (currentVersion === 0 && migration.version === 1) {
661
- db.prepare('INSERT INTO schema_version (version) VALUES (?)').run(migration.version);
662
- }
663
- else {
664
- db.prepare('UPDATE schema_version SET version = ?').run(migration.version);
665
- }
666
- })();
667
- currentVersion = migration.version;
668
- log.info('migration_applied', { version: migration.version, name: migration.name });
669
- }
670
- }