renor-engine 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +6 -6
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/BUILD.json +2 -2
  6. package/dist/api/server.js +10 -8
  7. package/dist/buildIdentity.js +18 -1
  8. package/dist/callPhases.js +82 -0
  9. package/dist/cli.js +10 -1
  10. package/dist/context/rowStore.js +1 -1
  11. package/dist/conversation/parser.js +40 -36
  12. package/dist/db/client.js +6 -4
  13. package/dist/db/handoffTable.js +31 -0
  14. package/dist/db/indexSalvage.js +24 -4
  15. package/dist/db/migrations.js +30 -2
  16. package/dist/db/sqlite.js +396 -0
  17. package/dist/digests/digestCache.js +6 -2
  18. package/dist/digests/digests.js +3 -3
  19. package/dist/digests/worker.js +3 -1
  20. package/dist/embeddings/caching.js +5 -5
  21. package/dist/embeddings/local.js +5 -3
  22. package/dist/embeddings/openai.js +4 -3
  23. package/dist/embeddings/simdScan.js +3 -1
  24. package/dist/embeddings/vectorIndex.js +0 -1
  25. package/dist/embeddings/vertex.js +123 -13
  26. package/dist/eval/cacheReport.js +7 -0
  27. package/dist/eval/context700-candidate.js +5 -5
  28. package/dist/eval/evalCache.js +46 -3
  29. package/dist/eval/indexSnapshots.js +548 -0
  30. package/dist/eval/pageDump.js +67 -0
  31. package/dist/eval/runManifest.js +6 -0
  32. package/dist/eval/textTokens.js +17 -0
  33. package/dist/evidence/attach.js +0 -3
  34. package/dist/evidence/evidenceCache.js +3 -3
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/preferenceExtractor.js +1 -4
  39. package/dist/extraction/typedFacts.js +1 -1
  40. package/dist/importing/apply.js +42 -14
  41. package/dist/importing/cli.js +9 -8
  42. package/dist/importing/existing.js +2 -0
  43. package/dist/importing/facts.js +0 -2
  44. package/dist/importing/plan.js +0 -5
  45. package/dist/importing/source.js +1 -1
  46. package/dist/indexer/parser.js +4 -1
  47. package/dist/indexer/pipeline.js +99 -279
  48. package/dist/indexer/walker.js +11 -25
  49. package/dist/mcp/server.js +36 -6
  50. package/dist/memory/handoff.js +363 -0
  51. package/dist/memory/lifecycle.js +310 -75
  52. package/dist/memory/sourceName.js +77 -1
  53. package/dist/memory/statusFacts.js +61 -0
  54. package/dist/memorymap/memoryMap.js +39 -13
  55. package/dist/memorymap/replacements.js +8 -3
  56. package/dist/mvp/agentMemory.js +106 -50
  57. package/dist/mvp/answerCheck.js +211 -0
  58. package/dist/mvp/cli.js +3 -1
  59. package/dist/mvp/compileCache.js +3 -10
  60. package/dist/mvp/compileSeed.js +9 -3
  61. package/dist/mvp/contextDistinct.js +53 -0
  62. package/dist/mvp/contextProfile.js +87 -0
  63. package/dist/mvp/directorySurface.js +68 -0
  64. package/dist/mvp/engine.js +39 -26
  65. package/dist/mvp/exportAll.js +17 -2
  66. package/dist/mvp/factMarkers.js +18 -2
  67. package/dist/mvp/history.js +5 -4
  68. package/dist/mvp/log.js +118 -0
  69. package/dist/mvp/server.js +642 -127
  70. package/dist/mvp/serverCard.js +12 -3
  71. package/dist/mvp/telemetry.js +35 -2
  72. package/dist/net/poolFetch.js +174 -0
  73. package/dist/recall/actionRecall.js +282 -0
  74. package/dist/recall/autoRecall.js +447 -0
  75. package/dist/recall/searchRecall.js +189 -0
  76. package/dist/retrieval/entityLookup.js +53 -4
  77. package/dist/retrieval/orchestrator.js +93 -8
  78. package/dist/retrieval/plainWording.js +236 -0
  79. package/dist/retrieval/readerLayout.js +143 -224
  80. package/dist/router/routerCache.js +2 -2
  81. package/dist/watcher/watcher.js +5 -3
  82. package/package.json +5 -7
  83. package/dist/importing/index.js +0 -29
@@ -36,6 +36,8 @@ const eventExtractor_1 = require("../extraction/eventExtractor");
36
36
  const preferenceExtractor_1 = require("../extraction/preferenceExtractor");
37
37
  const rowStore_1 = require("../context/rowStore");
38
38
  const laneRuns_1 = require("./laneRuns");
39
+ const log_1 = require("../mvp/log");
40
+ const log = (0, log_1.createLogger)('engine');
39
41
  /** Texts per request the embedding service takes at once (what Vertex's predict accepts). */
40
42
  const EMBED_REQUEST_TEXTS = 32;
41
43
  /**
@@ -61,7 +63,7 @@ const FACT_PREDICATES = ['owner', 'status', 'updated', 'tags'];
61
63
  const factText = (v) => (v !== null && typeof v === 'object' && !(v instanceof Date) ? JSON.stringify(v) : String(v));
62
64
  /** The note is indexed anyway; say which front-matter value was not text, so a note that seems to ignore it can be explained. */
63
65
  function warnFrontmatter(filePath, what) {
64
- console.warn(`[index] ${filePath}: ${what}; the note is indexed without it`);
66
+ log.warn('frontmatter_value_ignored', { note: path_1.default.basename(filePath), detail: `${what}; the note is indexed without it` });
65
67
  }
66
68
  /** The note's `kind`: a string, else none. Anything else (a mapping, a list, a date, a number) is ignored with a warning. */
67
69
  function noteKind(filePath, frontmatter) {
@@ -76,9 +78,9 @@ function noteKind(filePath, frontmatter) {
76
78
  * The extraction version each structured-row lane writes today. This is the
77
79
  * second half of the lane cache key, (source_hash, extraction_version), so it
78
80
  * is also the test for "is this preserved row still this lane's current row?".
79
- * One mapping, used by the in-transaction reattach and by the post-failure
80
- * restore, so a version bump cannot be applied in one place and missed in the
81
- * other. `row_kind` is CHECK-constrained to these three kinds.
81
+ * One mapping, used by the in-transaction reattach and by the lanes' cache
82
+ * check (laneDone), so a version bump cannot be applied in one place and missed
83
+ * in the other. `row_kind` is CHECK-constrained to these three kinds.
82
84
  */
83
85
  const LANE_EXTRACTION_VERSION = {
84
86
  fact: typedFacts_1.FACTS_EXTRACTION_VERSION,
@@ -88,6 +90,49 @@ const LANE_EXTRACTION_VERSION = {
88
90
  function currentVersion(kind) {
89
91
  return LANE_EXTRACTION_VERSION[kind];
90
92
  }
93
+ function lane(kind, label, extractor, payloadOf, fallbackText) {
94
+ return {
95
+ kind,
96
+ version: LANE_EXTRACTION_VERSION[kind],
97
+ label,
98
+ async extract(noteTitle, noteBody, chunks) {
99
+ const extracted = await extractor(noteTitle, noteBody);
100
+ return extracted.map((item) => {
101
+ const attributed = (0, writePathAdjudication_1.chunkForCandidate)(chunks, item);
102
+ return {
103
+ payload: payloadOf(item),
104
+ // The chunker gives every note a chunk; without one, 0 makes the row store refuse the row (ProvenanceError).
105
+ sourceChunkId: attributed?.chunkId ?? chunks[0]?.id ?? 0,
106
+ sourceText: attributed?.text ?? fallbackText(item),
107
+ };
108
+ });
109
+ },
110
+ };
111
+ }
112
+ /** The structured-row lanes this pass runs, in order: typed facts (C2), chronicle events (C3), standing preferences (C4). */
113
+ function structuredLanes(options) {
114
+ const lanes = [];
115
+ if (options.numericFactExtractor && (0, typedFacts_1.typedFactsEnabled)()) {
116
+ lanes.push(lane('fact', 'typed-fact', options.numericFactExtractor, f => ({ value: f.value, unit: f.unit, verb: f.verb, object: f.object, event: f.event }), f => `${f.value} ${f.object}`.trim()));
117
+ }
118
+ if (options.eventExtractor && (0, eventExtractor_1.chronicleEnabled)()) {
119
+ lanes.push(lane('event', 'event', options.eventExtractor, e => ({ when: e.when, event: e.event, entities: e.entities }), e => `${e.when} ${e.event}`.trim()));
120
+ }
121
+ if (options.preferenceExtractor && (0, preferenceExtractor_1.prefsEnabled)()) {
122
+ lanes.push(lane('preference', 'preference', options.preferenceExtractor, p => ({ statement: p.statement, kind: p.kind }), p => p.statement));
123
+ }
124
+ return lanes;
125
+ }
126
+ /**
127
+ * Whether a lane is done for these bytes, so it costs no model call. Either half of the cache key (source_hash,
128
+ * extraction_version) says so: a current row of the lane for this hash, or a completed zero-yield run (#189, lane_runs:
129
+ * a lane that produced nothing has no row to record it).
130
+ */
131
+ function laneDone(db, filePath, lane, noteHash) {
132
+ return !!(0, statements_1.prepareOnce)(db, `SELECT id FROM structured_rows
133
+ WHERE source_path = ? AND row_kind = ? AND extraction_version = ? AND source_hash = ? AND valid_to IS NULL LIMIT 1`).get(filePath, lane.kind, lane.version, noteHash)
134
+ || !!(0, laneRuns_1.laneZeroYieldMarker)(db, filePath, lane.kind, noteHash, lane.version);
135
+ }
91
136
  /**
92
137
  * Whether the vectors this index holds are in the embedding space the provider writes, so a stored vector may
93
138
  * stand in for a fresh one. A new index takes the provider's space; vectors recorded in a different one are
@@ -335,17 +380,17 @@ async function storeVectors(db, filePath, options, pass) {
335
380
  const texts = toEmbed.map(c => c.text);
336
381
  if (texts.length > 0) {
337
382
  const embeddings = await embeddingProvider.embed(texts, options.embedBudgetMs === undefined ? undefined : { budgetMs: options.embedBudgetMs * Math.ceil(texts.length / EMBED_REQUEST_TEXTS) });
338
- // Fix #3: Warn on embedding count mismatch instead of silently losing vectors
339
- if (embeddings.length !== texts.length) {
340
- console.error(`[pipeline] embedding count mismatch: expected ${texts.length}, got ${embeddings.length} for ${filePath}`);
383
+ // A vector missing from the answer fails the file: its hash stays '' (see transactionHash), so the next pass
384
+ // embeds the note again. Committing the hash here would leave those chunks without a vector for good, since an
385
+ // unchanged note is never read again.
386
+ if (embeddings.length !== texts.length || texts.some((_, i) => !embeddings[i])) {
387
+ throw new Error(`embedding service answered ${embeddings.length} vectors for ${texts.length} chunks of ${filePath}`);
341
388
  }
342
389
  for (let i = 0; i < toEmbed.length; i++) {
343
- if (embeddings[i]) {
344
- if (vectorIndex.insertNew)
345
- vectorIndex.insertNew(toEmbed[i].id, embeddings[i]);
346
- else
347
- vectorIndex.upsert(toEmbed[i].id, embeddings[i]);
348
- }
390
+ if (vectorIndex.insertNew)
391
+ vectorIndex.insertNew(toEmbed[i].id, embeddings[i]);
392
+ else
393
+ vectorIndex.upsert(toEmbed[i].id, embeddings[i]);
349
394
  }
350
395
  }
351
396
  // Fix #2 (cont): Commit the real note_hash only after embedding succeeds
@@ -572,37 +617,11 @@ async function indexFileUnlocked(db, filePath, options = {}) {
572
617
  // The facts of an unchanged note are not rebuilt below, so a row an earlier build made of the engine's own header
573
618
  // would stay. Remove it here: one statement for a daily note, no chunk or extractor involved.
574
619
  removeDailyHeaderFacts(db, filePath, content);
575
- // C2: unchanged content skips extraction ONLY when current-version rows
576
- // already exist (ADR §4.2 — the cache key is (source_hash,
577
- // extraction_version)). An extraction_version change invalidates the cache
578
- // and falls through to re-extract, still with zero model calls until the
579
- // (re-)extraction itself.
580
- if ((options.numericFactExtractor && (0, typedFacts_1.typedFactsEnabled)()) ||
581
- (options.eventExtractor && (0, eventExtractor_1.chronicleEnabled)()) ||
582
- (options.preferenceExtractor && (0, preferenceExtractor_1.prefsEnabled)())) {
583
- // The cache key is per lane: (source_hash, extraction_version). Skip
584
- // only when EVERY enabled lane is already done for these bytes — an
585
- // enabled lane without rows (first run, version bump) falls through.
586
- // "Done" is either half of the key: a current row for this hash, or a
587
- // completed zero-yield run (#189, lane_runs — a lane that produced
588
- // nothing has no row to record it).
589
- const factsEnabled = !!(options.numericFactExtractor && (0, typedFacts_1.typedFactsEnabled)());
590
- const eventsEnabled = !!(options.eventExtractor && (0, eventExtractor_1.chronicleEnabled)());
591
- const prefsEnabledLane = !!(options.preferenceExtractor && (0, preferenceExtractor_1.prefsEnabled)());
592
- const haveCurrentVersionRows = !factsEnabled || !!db.prepare(`SELECT id FROM structured_rows
593
- WHERE source_path = ? AND row_kind = 'fact' AND extraction_version = ? AND source_hash = ? AND valid_to IS NULL LIMIT 1`).get(filePath, typedFacts_1.FACTS_EXTRACTION_VERSION, noteHash) || !!(0, laneRuns_1.laneZeroYieldMarker)(db, filePath, 'fact', noteHash, typedFacts_1.FACTS_EXTRACTION_VERSION);
594
- const haveCurrentEventRows = !eventsEnabled || !!db.prepare(`SELECT id FROM structured_rows
595
- WHERE source_path = ? AND row_kind = 'event' AND extraction_version = ? AND source_hash = ? AND valid_to IS NULL LIMIT 1`).get(filePath, eventExtractor_1.EVENTS_EXTRACTION_VERSION, noteHash) || !!(0, laneRuns_1.laneZeroYieldMarker)(db, filePath, 'event', noteHash, eventExtractor_1.EVENTS_EXTRACTION_VERSION);
596
- const haveCurrentPrefRows = !prefsEnabledLane || !!db.prepare(`SELECT id FROM structured_rows
597
- WHERE source_path = ? AND row_kind = 'preference' AND extraction_version = ? AND source_hash = ? AND valid_to IS NULL LIMIT 1`).get(filePath, preferenceExtractor_1.PREFS_EXTRACTION_VERSION, noteHash) || !!(0, laneRuns_1.laneZeroYieldMarker)(db, filePath, 'preference', noteHash, preferenceExtractor_1.PREFS_EXTRACTION_VERSION);
598
- if (haveCurrentVersionRows && haveCurrentEventRows && haveCurrentPrefRows) {
599
- return; // unchanged content: zero model calls
600
- }
601
- // else: a lane lacks current rows → fall through, stale rows deleted below.
602
- }
603
- else {
604
- return; // unchanged — skip
605
- }
620
+ // C2: unchanged content skips extraction ONLY when every armed lane is done for these bytes (ADR §4.2: the
621
+ // cache key is (source_hash, extraction_version), see laneDone). A lane without rows (first run, version bump)
622
+ // falls through to re-extract; with no lane armed the note is simply skipped.
623
+ if (structuredLanes(options).every(lane => laneDone(db, filePath, lane, noteHash)))
624
+ return;
606
625
  }
607
626
  // 4. Parse note
608
627
  const detailed = (0, parser_1.parseNoteDetailed)(filePath, content);
@@ -660,13 +679,6 @@ async function indexFileUnlocked(db, filePath, options = {}) {
660
679
  FROM structured_rows r
661
680
  LEFT JOIN chunks c ON c.id = r.source_chunk_id
662
681
  WHERE r.source_path = ?`).all(filePath);
663
- const preservedByKind = new Map();
664
- for (const saved of preservedStructured) {
665
- const kind = String(saved.row_kind);
666
- const bucket = preservedByKind.get(kind) ?? [];
667
- bucket.push(saved);
668
- preservedByKind.set(kind, bucket);
669
- }
670
682
  // The chunks from the top of an edited note that are already stored exactly as they are now
671
683
  // keep their rows, search entries and vectors; only the rest is replaced (a ledger appended to
672
684
  // replaces its last event and adds the new one). The whole note is replaced instead when the
@@ -746,41 +758,12 @@ async function indexFileUnlocked(db, filePath, options = {}) {
746
758
  oldChunkIds: oldChunks.slice(keep).map(old => old.id),
747
759
  vectorSource,
748
760
  entityId,
749
- preservedByKind,
750
761
  cut,
751
762
  chunkHashes,
752
763
  };
753
764
  });
754
765
  // Fix #1 (cont): Delete old vectors only after transaction commits successfully
755
- const { insertedChunks, kept, oldChunkIds, vectorSource, entityId, preservedByKind, cut, chunkHashes } = upsertAll();
756
- // These post-failure helpers and the in-transaction reattach above must
757
- // agree on one thing: a preserved row only counts as that lane's current row
758
- // when its extraction_version matches the version that lane writes today
759
- // (currentVersion), which is the same key the lane cache-skip below uses.
760
- function preservedLaneMatches(kind) {
761
- const savedRows = preservedByKind.get(kind) ?? [];
762
- return savedRows.some(saved => (saved.valid_to == null
763
- && saved.source_hash === noteHash
764
- && saved.extraction_version === currentVersion(kind)));
765
- }
766
- function restorePreservedLane(kind) {
767
- const savedRows = (preservedByKind.get(kind) ?? []).filter(saved => (saved.valid_to == null
768
- && saved.source_hash === noteHash
769
- && saved.extraction_version === currentVersion(kind)));
770
- if (savedRows.length === 0 || insertedChunks.length === 0)
771
- return;
772
- const restore = db.prepare(`INSERT INTO structured_rows
773
- (row_kind, source_path, source_hash, source_chunk_id, scope, extraction_version,
774
- valid_from, valid_to, payload_json, dedupe_key, confirm_count, created_at)
775
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
776
- for (const saved of savedRows) {
777
- const sourceText = typeof saved.source_text === 'string' ? saved.source_text : '';
778
- const attributed = sourceText
779
- ? (0, writePathAdjudication_1.chunkForCandidate)(insertedChunks, { statement: sourceText })
780
- : null;
781
- restore.run(kind, filePath, saved.source_hash, attributed?.chunkId ?? insertedChunks[0].id, saved.scope, saved.extraction_version, saved.valid_from, saved.valid_to, saved.payload_json, saved.dedupe_key, saved.confirm_count, saved.created_at);
782
- }
783
- }
766
+ const { insertedChunks, kept, oldChunkIds, vectorSource, entityId, cut, chunkHashes } = upsertAll();
784
767
  await storeVectors(db, filePath, options, { insertedChunks, kept, oldChunkIds, vectorSource, vectorsMatch, embedsHere, noteHash });
785
768
  // 11b. LLM fact extraction — runs outside transaction, after embeddings (async/network).
786
769
  // Fix #5: On extraction failure, reset note_hash to '' so the next unchanged-file
@@ -802,7 +785,7 @@ async function indexFileUnlocked(db, filePath, options = {}) {
802
785
  }
803
786
  }
804
787
  catch (err) {
805
- console.error(`[pipeline] LLM extraction failed for ${filePath}:`, err.message);
788
+ log.error('extraction_failed', { lane: 'LLM', note: path_1.default.basename(filePath) }, err);
806
789
  // #188: reset the hash so the next pass retries (Fix #5), AND rethrow.
807
790
  // Swallowing it here left `indexVault` counting the note as INDEXED, so
808
791
  // `index` reported 0 errors and exited 0 with this note's facts missing —
@@ -833,7 +816,7 @@ async function indexFileUnlocked(db, filePath, options = {}) {
833
816
  }, options.decisionModel);
834
817
  }
835
818
  catch (err) {
836
- console.error(`[pipeline] facts write-path capture failed for ${filePath}:`, err.message);
819
+ log.error('extraction_failed', { lane: 'facts write-path capture', note: path_1.default.basename(filePath) }, err);
837
820
  // Same #188 contract as the legacy lane above: reset the hash so the next
838
821
  // pass retries, then rethrow so the failure is counted and reaches the
839
822
  // exit status instead of being hidden by a blanket catch.
@@ -841,201 +824,38 @@ async function indexFileUnlocked(db, filePath, options = {}) {
841
824
  throw err;
842
825
  }
843
826
  }
844
- // 11c. C2 typed numeric facts → structured_rows (RENOR_TYPED_FACTS, default
845
- // off; ADR-0003 §4 provenance contract + §4.2 cache invalidation). The
846
- // (source_hash, extraction_version) skip happens BEFORE any model call:
847
- // unchanged content is either already covered by current-version rows or by
848
- // a completed zero-yield run (lane_runs, #189) and costs zero extractor calls.
849
- // Edit path: stale rows for this source are deleted and re-extracted. On
850
- // extractor-parse-error the note hash resets so the next pass retries;
851
- // parse failures are never cached and never silently [].
852
- // HARMONIZATION: candidates flow through the DecisionModel seam
853
- // (adjudicateAndWriteRows) BEFORE any row is written — capture-gate /
854
- // sensitivity-guardrail (a `sensitive` decision blocks the candidate) and
855
- // dedupe adjudication with the confirm-count + v11 unique-index semantics.
856
- if (options.numericFactExtractor && (0, typedFacts_1.typedFactsEnabled)()) {
857
- // One predicate, both halves of the lane cache: a current row for these
858
- // bytes, OR a completed zero-yield run for them (#189). A lane that
859
- // produced nothing must be recognised BEFORE any extraction work starts,
860
- // or it is re-extracted on every unchanged pass forever.
861
- const structured = db.prepare(`SELECT id FROM structured_rows
862
- WHERE source_path = ? AND row_kind = 'fact' AND extraction_version = ? AND source_hash = ? AND valid_to IS NULL LIMIT 1`).get(filePath, typedFacts_1.FACTS_EXTRACTION_VERSION, noteHash)
863
- ?? (0, laneRuns_1.laneZeroYieldMarker)(db, filePath, 'fact', noteHash, typedFacts_1.FACTS_EXTRACTION_VERSION);
864
- if (!structured) {
865
- // This lane is stale. Re-extract only fact rows; a current fact lane
866
- // falls through so events and preferences still run. The previous rows
867
- // stay until the new extraction returns, so an extractor that yields
868
- // nothing does not empty the lane. An extractor that THROWS does: a
869
- // stale-version lane is left empty on purpose (see the catch below).
870
- try {
871
- const extracted = await options.numericFactExtractor(parsed.title, parsed.body);
872
- (0, rowStore_1.deleteStructuredRowsForSourceKind)(db, filePath, 'fact');
873
- const validFrom = (0, typedFacts_1.contentDateFromFrontmatter)(parsed.frontmatter);
874
- const firstChunkId = db.prepare('SELECT id FROM chunks WHERE note_path = ? ORDER BY id LIMIT 1').get(filePath)?.id;
875
- const candidates = extracted.map((f) => {
876
- const attributed = (0, writePathAdjudication_1.chunkForCandidate)(insertedChunks, f);
877
- return {
878
- payload: { value: f.value, unit: f.unit, verb: f.verb, object: f.object, event: f.event },
879
- sourceChunkId: attributed?.chunkId ?? firstChunkId ?? 1,
880
- sourceText: attributed?.text ?? `${f.value} ${f.object}`.trim(),
881
- };
882
- });
883
- const outcome = await (0, writePathAdjudication_1.adjudicateAndWriteRows)(db, {
884
- rowKind: 'fact',
885
- extractionVersion: typedFacts_1.FACTS_EXTRACTION_VERSION,
886
- notePath: filePath,
887
- sourceHash: noteHash,
888
- validFrom,
889
- candidates,
890
- }, options.decisionModel);
891
- // #189: the zero-yield checkpoint. This lane used to be called again
892
- // on every unchanged pass because a zero yield leaves no row to mark it
893
- // current; the completed run is recorded here instead — after the
894
- // extractor and the writes both succeeded, so the throw path below
895
- // stays uncached (it resets the note hash and must retry). What is
896
- // recorded is what THIS run's writes returned — a fact of the run, not a
897
- // count re-read from structured_rows, which a concurrent writer could
898
- // tear (review B1) — so a zero is exactly "left nothing current".
899
- (0, laneRuns_1.recordLaneRun)(db, filePath, 'fact', noteHash, typedFacts_1.FACTS_EXTRACTION_VERSION, outcome.rowsWritten);
900
- }
901
- catch (err) {
902
- console.error(`[pipeline] typed-fact extraction failed for ${filePath}:`, err.message);
903
- // Never cached, never silent []: reset the note hash so the next pass
904
- // retries, then rethrow the explicit error (extractor-parse-error or
905
- // otherwise) — a note whose extraction failed is never reported current.
906
- // A stale-version row was not reattached and is deliberately NOT put
907
- // back: a bumped extraction_version invalidates the lane, so its old
908
- // payload must never be served as current. The lane stays empty until
909
- // the retry succeeds, which the reset hash guarantees it will attempt.
910
- db.prepare("UPDATE notes SET note_hash = '' WHERE path = ?").run(filePath);
911
- // An edited note must not keep the previous assertion as current
912
- // against the new text. Restore only when the preserved row is still
913
- // this lane's current row for these bytes, including a retry after an
914
- // earlier failure cleared the note hash.
915
- if (noteUnchanged || preservedLaneMatches('fact'))
916
- restorePreservedLane('fact');
917
- throw err;
918
- }
919
- }
920
- }
921
- // 11d. C3 chronicle events → structured_rows (RENOR_CHRONICLE, default off;
922
- // ADR-0003 §4 event-lane payload + §4.1 identity-keyed supersession via the
923
- // shared rowStore insert). The (source_hash, extraction_version) skip happens
924
- // BEFORE any model call: unchanged content is either already covered by
925
- // current-version event rows or by a completed zero-yield run (lane_runs,
926
- // #189) and costs zero extractor calls. Edit path: stale rows for this source are
927
- // deleted and re-extracted. `when` stays the user's own wording inside the
928
- // payload; the row's valid_from comes only from the note's frontmatter date
929
- // (NULL = unknown, never resolved/invented from the relative `when`). On
930
- // extractor-parse-error the note hash resets so the next pass retries;
931
- // parse failures are never cached and never silently [].
932
- // HARMONIZATION: candidates flow through the DecisionModel seam
933
- // (adjudicateAndWriteRows) BEFORE any row is written — capture-gate /
934
- // sensitivity-guardrail (a `sensitive` decision blocks the candidate) and
935
- // dedupe adjudication; the write keeps the fleet's identity-keyed
936
- // supersession semantics (newer dated restatement supersedes; NULL dates
937
- // never supersede).
938
- if (options.eventExtractor && (0, eventExtractor_1.chronicleEnabled)()) {
939
- const currentEvents = db.prepare(`SELECT id FROM structured_rows
940
- WHERE source_path = ? AND row_kind = 'event' AND extraction_version = ? AND source_hash = ? AND valid_to IS NULL LIMIT 1`).get(filePath, eventExtractor_1.EVENTS_EXTRACTION_VERSION, noteHash)
941
- ?? (0, laneRuns_1.laneZeroYieldMarker)(db, filePath, 'event', noteHash, eventExtractor_1.EVENTS_EXTRACTION_VERSION);
942
- if (!currentEvents) {
943
- // This lane is stale. Re-extract only event rows. The previous rows
944
- // stay until the new extraction returns.
945
- try {
946
- const extracted = await options.eventExtractor(parsed.title, parsed.body);
947
- (0, rowStore_1.deleteStructuredRowsForSourceKind)(db, filePath, 'event');
948
- const validFrom = (0, typedFacts_1.contentDateFromFrontmatter)(parsed.frontmatter);
949
- const firstChunkId = db.prepare('SELECT id FROM chunks WHERE note_path = ? ORDER BY id LIMIT 1').get(filePath)?.id;
950
- const candidates = extracted.map((e) => {
951
- const attributed = (0, writePathAdjudication_1.chunkForCandidate)(insertedChunks, e);
952
- return {
953
- payload: { when: e.when, event: e.event, entities: e.entities },
954
- sourceChunkId: attributed?.chunkId ?? firstChunkId ?? 1,
955
- sourceText: attributed?.text ?? `${e.when} ${e.event}`.trim(),
956
- };
957
- });
958
- const outcome = await (0, writePathAdjudication_1.adjudicateAndWriteRows)(db, {
959
- rowKind: 'event',
960
- extractionVersion: eventExtractor_1.EVENTS_EXTRACTION_VERSION,
961
- notePath: filePath,
962
- sourceHash: noteHash,
963
- validFrom,
964
- candidates,
965
- }, options.decisionModel);
966
- // Zero-yield events: same checkpoint as the fact lane (#189).
967
- (0, laneRuns_1.recordLaneRun)(db, filePath, 'event', noteHash, eventExtractor_1.EVENTS_EXTRACTION_VERSION, outcome.rowsWritten);
968
- }
969
- catch (err) {
970
- console.error(`[pipeline] event extraction failed for ${filePath}:`, err.message);
971
- // Never cached, never silent []: reset the note hash so the next pass
972
- // retries, then rethrow the explicit error (extractor-parse-error or
973
- // otherwise) — a note whose extraction failed is never reported current.
974
- db.prepare("UPDATE notes SET note_hash = '' WHERE path = ?").run(filePath);
975
- if (noteUnchanged || preservedLaneMatches('event'))
976
- restorePreservedLane('event');
977
- throw err;
978
- }
827
+ // 11c-e. The structured-row lanes (typed facts, chronicle events, standing preferences; each flag-gated, default
828
+ // off; ADR-0003 §4). A lane already done for these bytes (see laneDone) costs no model call. Otherwise its
829
+ // previous rows stay until the extractor returns, so an extractor that yields nothing does not empty the lane;
830
+ // the candidates then flow through the DecisionModel seam (adjudicateAndWriteRows: capture gate, sensitivity
831
+ // guardrail, dedupe, identity-keyed supersession) before any row is written, and the completed run is recorded
832
+ // (#189: a zero-yield lane has no row to mark it done).
833
+ //
834
+ // A failure (extractor-parse-error or otherwise) is never cached and never a silent []: the note hash resets so
835
+ // the next pass retries, and the error is rethrown so it is counted (#188). A row of a stale version was not
836
+ // reattached above and is deliberately not put back: a bumped extraction_version must never be served as current.
837
+ for (const lane of structuredLanes(options)) {
838
+ if (laneDone(db, filePath, lane, noteHash))
839
+ continue;
840
+ try {
841
+ const candidates = await lane.extract(parsed.title, parsed.body, insertedChunks);
842
+ (0, rowStore_1.deleteStructuredRowsForSourceKind)(db, filePath, lane.kind);
843
+ const outcome = await (0, writePathAdjudication_1.adjudicateAndWriteRows)(db, {
844
+ rowKind: lane.kind,
845
+ extractionVersion: lane.version,
846
+ notePath: filePath,
847
+ sourceHash: noteHash,
848
+ // The note's front-matter date (NULL = unknown), never one resolved from an extracted phrase.
849
+ validFrom: (0, typedFacts_1.contentDateFromFrontmatter)(parsed.frontmatter),
850
+ candidates,
851
+ }, options.decisionModel);
852
+ // What this run's writes returned, not a re-read of structured_rows a concurrent writer could tear (review B1).
853
+ (0, laneRuns_1.recordLaneRun)(db, filePath, lane.kind, noteHash, lane.version, outcome.rowsWritten);
979
854
  }
980
- }
981
- // 11e. C4 standing preferences → structured_rows (RENOR_PREFS, default off;
982
- // ADR-0003 §4 preference-lane payload + §4.1 identity-keyed supersession via
983
- // the shared rowStore insert — contradiction on edit supersedes, never
984
- // co-current, only via (row_kind, scope, norm(statement)) identity). The
985
- // (source_hash, extraction_version) skip happens BEFORE any model call:
986
- // unchanged content is either already covered by current-version pref rows
987
- // or by a completed zero-yield run (lane_runs, #189) and costs zero
988
- // extractor calls. `kind` is an unvalidated render-only enum, never in the
989
- // dedupe key. One-off past-tense statements are dropped by the extractor's
990
- // structural guard. On extractor-parse-error the note hash resets so the
991
- // next pass retries; parse failures are never cached and never silently [].
992
- // HARMONIZATION: candidates flow through the DecisionModel seam
993
- // (adjudicateAndWriteRows) BEFORE any row is written — capture-gate /
994
- // sensitivity-guardrail (a `sensitive` decision blocks the candidate) and
995
- // dedupe adjudication (norm(statement) identity); the write keeps the
996
- // fleet's identity-keyed supersession semantics (contradiction on edit →
997
- // same-identity newer restatement supersedes, never co-current).
998
- if (options.preferenceExtractor && (0, preferenceExtractor_1.prefsEnabled)()) {
999
- const currentPrefs = db.prepare(`SELECT id FROM structured_rows
1000
- WHERE source_path = ? AND row_kind = 'preference' AND extraction_version = ? AND source_hash = ? AND valid_to IS NULL LIMIT 1`).get(filePath, preferenceExtractor_1.PREFS_EXTRACTION_VERSION, noteHash)
1001
- ?? (0, laneRuns_1.laneZeroYieldMarker)(db, filePath, 'preference', noteHash, preferenceExtractor_1.PREFS_EXTRACTION_VERSION);
1002
- if (!currentPrefs) {
1003
- // This lane is stale. Re-extract only preference rows. The previous
1004
- // rows stay until the new extraction returns.
1005
- try {
1006
- const extracted = await options.preferenceExtractor(parsed.title, parsed.body);
1007
- (0, rowStore_1.deleteStructuredRowsForSourceKind)(db, filePath, 'preference');
1008
- const validFrom = (0, typedFacts_1.contentDateFromFrontmatter)(parsed.frontmatter);
1009
- const firstChunkId = db.prepare('SELECT id FROM chunks WHERE note_path = ? ORDER BY id LIMIT 1').get(filePath)?.id;
1010
- const candidates = extracted.map((p) => {
1011
- const attributed = (0, writePathAdjudication_1.chunkForCandidate)(insertedChunks, p);
1012
- return {
1013
- payload: { statement: p.statement, kind: p.kind },
1014
- sourceChunkId: attributed?.chunkId ?? firstChunkId ?? 1,
1015
- sourceText: attributed?.text ?? p.statement,
1016
- };
1017
- });
1018
- const outcome = await (0, writePathAdjudication_1.adjudicateAndWriteRows)(db, {
1019
- rowKind: 'preference',
1020
- extractionVersion: preferenceExtractor_1.PREFS_EXTRACTION_VERSION,
1021
- notePath: filePath,
1022
- sourceHash: noteHash,
1023
- validFrom,
1024
- candidates,
1025
- }, options.decisionModel);
1026
- // Zero-yield preferences: same checkpoint as the fact lane (#189).
1027
- (0, laneRuns_1.recordLaneRun)(db, filePath, 'preference', noteHash, preferenceExtractor_1.PREFS_EXTRACTION_VERSION, outcome.rowsWritten);
1028
- }
1029
- catch (err) {
1030
- console.error(`[pipeline] preference extraction failed for ${filePath}:`, err.message);
1031
- // Never cached, never silent []: reset the note hash so the next pass
1032
- // retries, then rethrow the explicit error (extractor-parse-error or
1033
- // otherwise) — a note whose extraction failed is never reported current.
1034
- db.prepare("UPDATE notes SET note_hash = '' WHERE path = ?").run(filePath);
1035
- if (noteUnchanged || preservedLaneMatches('preference'))
1036
- restorePreservedLane('preference');
1037
- throw err;
1038
- }
855
+ catch (err) {
856
+ log.error('extraction_failed', { lane: lane.label, note: path_1.default.basename(filePath) }, err);
857
+ db.prepare("UPDATE notes SET note_hash = '' WHERE path = ?").run(filePath);
858
+ throw err;
1039
859
  }
1040
860
  }
1041
861
  const rows = { ids: insertedChunks.map(c => c.id), hashes: chunkHashes };
@@ -1095,11 +915,11 @@ async function indexVault(db, vaultPath, options = {}, concurrency = 5) {
1095
915
  active--;
1096
916
  if (err instanceof NoteTooLargeError) {
1097
917
  skippedTooLarge.push(path_1.default.relative(vaultPath, filePath));
1098
- console.warn(`[index] ${filePath}: ${err.message}`);
918
+ log.warn('note_too_large', { file: path_1.default.relative(vaultPath, filePath) }, err);
1099
919
  }
1100
920
  else {
1101
921
  errors++;
1102
- console.error(`[index] error indexing ${filePath}:`, err);
922
+ log.error('index_file_failed', { file: path_1.default.relative(vaultPath, filePath) }, err);
1103
923
  }
1104
924
  onComplete();
1105
925
  });
@@ -24,20 +24,18 @@ function compileIgnore(patterns) {
24
24
  });
25
25
  return rel => rules.some(r => rel === r.clean || rel.startsWith(r.dir) || rel.includes(r.inside) || rel.endsWith(r.tail));
26
26
  }
27
- async function walkVault(vaultPath, options = {}) {
28
- let patterns = [...DEFAULT_IGNORE, ...(options.ignorePatterns ?? [])];
27
+ /** The default ignore patterns, then `extra`, then the lines of the vault's .semanticignore (blank and `#` lines skipped). */
28
+ async function vaultIgnorePatterns(vaultPath, extra = []) {
29
+ const patterns = [...DEFAULT_IGNORE, ...extra];
29
30
  try {
30
31
  const raw = await promises_1.default.readFile(path_1.default.join(vaultPath, '.semanticignore'), 'utf8');
31
- const fromFile = raw
32
- .split('\n')
33
- .map(l => l.trim())
34
- .filter(l => l && !l.startsWith('#'));
35
- patterns = [...patterns, ...fromFile];
36
- }
37
- catch {
38
- /* no .semanticignore */
32
+ patterns.push(...raw.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#')));
39
33
  }
40
- const ignored = compileIgnore(patterns);
34
+ catch { /* no .semanticignore */ }
35
+ return patterns;
36
+ }
37
+ async function walkVault(vaultPath, options = {}) {
38
+ const ignored = compileIgnore(await vaultIgnorePatterns(vaultPath, options.ignorePatterns));
41
39
  const results = [];
42
40
  /**
43
41
  * `relDir` is the folder's path below the vault. `raw` is the folder as bytes and `shown` its path below the
@@ -156,13 +154,7 @@ function statVaultFiles(files) {
156
154
  * .semanticignore once, when asked.
157
155
  */
158
156
  async function vaultIgnores(vaultPath) {
159
- let patterns = [...DEFAULT_IGNORE];
160
- try {
161
- const raw = await promises_1.default.readFile(path_1.default.join(vaultPath, '.semanticignore'), 'utf8');
162
- patterns = [...patterns, ...raw.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#'))];
163
- }
164
- catch { /* no .semanticignore */ }
165
- const ignored = compileIgnore(patterns);
157
+ const ignored = compileIgnore(await vaultIgnorePatterns(vaultPath));
166
158
  return (file) => {
167
159
  const parts = path_1.default.relative(vaultPath, file).split(path_1.default.sep);
168
160
  return parts.some((_, i) => ignored(parts.slice(0, i + 1).join(path_1.default.sep)));
@@ -174,11 +166,5 @@ async function vaultIgnores(vaultPath) {
174
166
  * moved through its path any more than it can be found by search.
175
167
  */
176
168
  async function isIgnoredByVault(vaultPath, rel) {
177
- let patterns = [...DEFAULT_IGNORE];
178
- try {
179
- const raw = await promises_1.default.readFile(path_1.default.join(vaultPath, '.semanticignore'), 'utf8');
180
- patterns = [...patterns, ...raw.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#'))];
181
- }
182
- catch { /* no .semanticignore */ }
183
- return compileIgnore(patterns)(rel.split('/').join(path_1.default.sep));
169
+ return compileIgnore(await vaultIgnorePatterns(vaultPath))(rel.split('/').join(path_1.default.sep));
184
170
  }
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.TOOLS = void 0;
3
+ exports.HANDOFF_TOOL = exports.TOOLS = void 0;
4
4
  exports.createMcpTools = createMcpTools;
5
5
  exports.utf8Lines = utf8Lines;
6
6
  exports.registerMcpHandlers = registerMcpHandlers;
@@ -27,6 +27,8 @@ const sourceName_1 = require("../memory/sourceName");
27
27
  const requestErrors_1 = require("./requestErrors");
28
28
  const sanitize_1 = require("../mvp/sanitize");
29
29
  const factsFirst_1 = require("../retrieval/factsFirst");
30
+ const handoff_1 = require("../memory/handoff");
31
+ const buildIdentity_1 = require("../buildIdentity");
30
32
  function createMcpTools(db, vectorIndex, provider, config, evidence = { mode: 'off', model: '', maxSessions: 12, charCap: 6000 }) {
31
33
  // Startup configuration, captured once: tool arguments and later env changes cannot rename a connection.
32
34
  // The hive name is the header form. Empty, unnamed, or a second live holder publishes no hive tools
@@ -66,8 +68,18 @@ function createMcpTools(db, vectorIndex, provider, config, evidence = { mode: 'o
66
68
  return { warning: 'memory-saved-but-index-refresh-failed', detail: detail.slice(0, 300) };
67
69
  }
68
70
  };
71
+ // The handoff note (memory/handoff.ts), only with RENOR_HANDOFF_NOTE on. Its writer is this connection's startup name;
72
+ // its expiry sweep runs for the life of the process (the timer never keeps it alive).
73
+ if ((0, handoff_1.handoffNoteEnabled)())
74
+ (0, handoff_1.startHandoffSweep)(db);
75
+ const handoff = (0, handoff_1.handoffNoteEnabled)()
76
+ ? { async [handoff_1.HANDOFF_TOOL_NAME](args) {
77
+ return (0, handoff_1.runHandoffTool)(db, args ?? {}, identity ?? 'stdio');
78
+ } }
79
+ : {};
69
80
  return {
70
81
  ...hive,
82
+ ...handoff,
71
83
  // Canonical tool names. Clients that prefix with the server name render
72
84
  // these as renor_search, renor_keyword, renor_facts, etc.
73
85
  async search(args) {
@@ -290,6 +302,21 @@ exports.TOOLS = [
290
302
  },
291
303
  },
292
304
  ];
305
+ /** Listed beside TOOLS only when the server has the handler, i.e. RENOR_HANDOFF_NOTE was on at startup. */
306
+ exports.HANDOFF_TOOL = {
307
+ name: handoff_1.HANDOFF_TOOL_NAME,
308
+ description: handoff_1.HANDOFF_TOOL_DESCRIPTION,
309
+ inputSchema: {
310
+ type: 'object',
311
+ properties: {
312
+ action: { type: 'string', enum: ['read', 'write'], description: handoff_1.HANDOFF_ARG_DESCRIPTIONS.action },
313
+ repo: { type: 'string', description: handoff_1.HANDOFF_ARG_DESCRIPTIONS.repo },
314
+ branch: { type: 'string', description: handoff_1.HANDOFF_ARG_DESCRIPTIONS.branch },
315
+ note: { type: 'string', description: handoff_1.HANDOFF_ARG_DESCRIPTIONS.note },
316
+ },
317
+ required: ['action', 'repo', 'branch'],
318
+ },
319
+ };
293
320
  /**
294
321
  * The stdin of the stdio server as lines the MCP transport can read, with a line that is not valid UTF-8 kept out.
295
322
  *
@@ -338,11 +365,14 @@ function utf8Lines(source, onInvalid) {
338
365
  * the published schema or the error envelope the client actually sees were wrong.
339
366
  */
340
367
  function registerMcpHandlers(server, tools) {
341
- const hive = hive_1.HIVE_TOOLS.filter(tool => typeof tools[tool.name] === 'function');
342
- server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => ({ tools: [...exports.TOOLS, ...hive] }));
368
+ // Looked up by the name the client sent: an argument shape is the tool's own to check.
369
+ const byName = tools;
370
+ const hive = hive_1.HIVE_TOOLS.filter(tool => typeof byName[tool.name] === 'function');
371
+ const handoff = typeof byName[handoff_1.HANDOFF_TOOL_NAME] === 'function' ? [exports.HANDOFF_TOOL] : [];
372
+ server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => ({ tools: [...exports.TOOLS, ...hive, ...handoff] }));
343
373
  server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
344
374
  const { name, arguments: args } = request.params;
345
- const handler = tools[name];
375
+ const handler = byName[name];
346
376
  if (!handler) {
347
377
  if (hive_1.HIVE_TOOLS.some(tool => tool.name === name)) {
348
378
  return { content: [{ type: 'text', text: `${name} is refused: this connection has no name. Set RENOR_CONNECTION_NAME to this agent's name before starting renor-engine mcp, then reconnect.` }], isError: true };
@@ -357,12 +387,12 @@ function registerMcpHandlers(server, tools) {
357
387
  return { content: [{ type: 'text', text: (0, sanitize_1.escapeInvisibleControls)(JSON.stringify(result)) }] };
358
388
  }
359
389
  catch (err) {
360
- return { content: [{ type: 'text', text: (0, sanitize_1.escapeInvisibleControls)(`Error: ${err.message}`) }], isError: true };
390
+ return { content: [{ type: 'text', text: (0, sanitize_1.escapeInvisibleControls)(`Error: ${err instanceof Error ? err.message : String(err)}`) }], isError: true };
361
391
  }
362
392
  });
363
393
  }
364
394
  async function startMcpServer(db, vectorIndex, provider, config) {
365
- const server = new index_js_1.Server({ name: 'renor', version: '1.0.0' }, { capabilities: { tools: {} } });
395
+ const server = new index_js_1.Server({ name: 'renor', version: buildIdentity_1.ENGINE_VERSION }, { capabilities: { tools: {} } });
366
396
  const evidence = (0, options_1.resolveEvidenceToolOptions)(config);
367
397
  const tools = createMcpTools(db, vectorIndex, provider, config, evidence);
368
398
  registerMcpHandlers(server, tools);