renor-engine 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +6 -6
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/BUILD.json +2 -2
  6. package/dist/api/server.js +10 -8
  7. package/dist/buildIdentity.js +18 -1
  8. package/dist/callPhases.js +82 -0
  9. package/dist/cli.js +10 -1
  10. package/dist/context/rowStore.js +1 -1
  11. package/dist/conversation/parser.js +40 -36
  12. package/dist/db/client.js +6 -4
  13. package/dist/db/handoffTable.js +31 -0
  14. package/dist/db/indexSalvage.js +24 -4
  15. package/dist/db/migrations.js +30 -2
  16. package/dist/db/sqlite.js +396 -0
  17. package/dist/digests/digestCache.js +6 -2
  18. package/dist/digests/digests.js +3 -3
  19. package/dist/digests/worker.js +3 -1
  20. package/dist/embeddings/caching.js +5 -5
  21. package/dist/embeddings/local.js +5 -3
  22. package/dist/embeddings/openai.js +4 -3
  23. package/dist/embeddings/simdScan.js +3 -1
  24. package/dist/embeddings/vectorIndex.js +0 -1
  25. package/dist/embeddings/vertex.js +123 -13
  26. package/dist/eval/cacheReport.js +7 -0
  27. package/dist/eval/context700-candidate.js +5 -5
  28. package/dist/eval/evalCache.js +46 -3
  29. package/dist/eval/indexSnapshots.js +548 -0
  30. package/dist/eval/pageDump.js +67 -0
  31. package/dist/eval/runManifest.js +6 -0
  32. package/dist/eval/textTokens.js +17 -0
  33. package/dist/evidence/attach.js +0 -3
  34. package/dist/evidence/evidenceCache.js +3 -3
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/preferenceExtractor.js +1 -4
  39. package/dist/extraction/typedFacts.js +1 -1
  40. package/dist/importing/apply.js +42 -14
  41. package/dist/importing/cli.js +9 -8
  42. package/dist/importing/existing.js +2 -0
  43. package/dist/importing/facts.js +0 -2
  44. package/dist/importing/plan.js +0 -5
  45. package/dist/importing/source.js +1 -1
  46. package/dist/indexer/parser.js +4 -1
  47. package/dist/indexer/pipeline.js +99 -279
  48. package/dist/indexer/walker.js +11 -25
  49. package/dist/mcp/server.js +36 -6
  50. package/dist/memory/handoff.js +363 -0
  51. package/dist/memory/lifecycle.js +310 -75
  52. package/dist/memory/sourceName.js +77 -1
  53. package/dist/memory/statusFacts.js +61 -0
  54. package/dist/memorymap/memoryMap.js +39 -13
  55. package/dist/memorymap/replacements.js +8 -3
  56. package/dist/mvp/agentMemory.js +106 -50
  57. package/dist/mvp/answerCheck.js +211 -0
  58. package/dist/mvp/cli.js +3 -1
  59. package/dist/mvp/compileCache.js +3 -10
  60. package/dist/mvp/compileSeed.js +9 -3
  61. package/dist/mvp/contextDistinct.js +53 -0
  62. package/dist/mvp/contextProfile.js +87 -0
  63. package/dist/mvp/directorySurface.js +68 -0
  64. package/dist/mvp/engine.js +39 -26
  65. package/dist/mvp/exportAll.js +17 -2
  66. package/dist/mvp/factMarkers.js +18 -2
  67. package/dist/mvp/history.js +5 -4
  68. package/dist/mvp/log.js +118 -0
  69. package/dist/mvp/server.js +642 -127
  70. package/dist/mvp/serverCard.js +12 -3
  71. package/dist/mvp/telemetry.js +35 -2
  72. package/dist/net/poolFetch.js +174 -0
  73. package/dist/recall/actionRecall.js +282 -0
  74. package/dist/recall/autoRecall.js +447 -0
  75. package/dist/recall/searchRecall.js +189 -0
  76. package/dist/retrieval/entityLookup.js +53 -4
  77. package/dist/retrieval/orchestrator.js +93 -8
  78. package/dist/retrieval/plainWording.js +236 -0
  79. package/dist/retrieval/readerLayout.js +143 -224
  80. package/dist/router/routerCache.js +2 -2
  81. package/dist/watcher/watcher.js +5 -3
  82. package/package.json +5 -7
  83. package/dist/importing/index.js +0 -29
@@ -39,6 +39,8 @@ const rebuild_1 = require("../embeddings/rebuild");
39
39
  const openai_1 = require("../embeddings/openai");
40
40
  const local_1 = require("../embeddings/local");
41
41
  const vertex_1 = require("../embeddings/vertex");
42
+ const poolFetch_1 = require("../net/poolFetch");
43
+ const callPhases_1 = require("../callPhases");
42
44
  const caching_1 = require("../embeddings/caching");
43
45
  const interactive_1 = require("../embeddings/interactive");
44
46
  const coalescing_1 = require("../embeddings/coalescing");
@@ -65,6 +67,8 @@ const contextRelevance_1 = require("./contextRelevance");
65
67
  const sanitize_1 = require("./sanitize");
66
68
  const digests_1 = require("../digests/digests");
67
69
  const worker_1 = require("../digests/worker");
70
+ const log_1 = require("./log");
71
+ const log = (0, log_1.createLogger)('engine');
68
72
  exports.DEFAULT_EMBED_BUDGETS = { writeMs: 5_000, readMs: 3_000 };
69
73
  /**
70
74
  * `RENOR_EMBED_WRITE_BUDGET_MS` and `RENOR_EMBED_READ_BUDGET_MS` set the two budgets; `0` for either (or both) lets
@@ -145,7 +149,13 @@ function withQueryCache(provider, config, scope) {
145
149
  }
146
150
  function createEmbeddingProvider(config,
147
151
  /** A test points the Vertex endpoint at a local stub and shortens its retry waits; production passes nothing. */
148
- vertexSeam) {
152
+ vertexSeam,
153
+ /**
154
+ * The service keeps its connection to the embedding service across the pauses between two reads, up to three minutes
155
+ * (src/net/poolFetch.ts). A bench or a command that builds the stack for one run leaves it off: it makes its requests
156
+ * back to back and gains nothing from a pool that would outlive it, and it keeps the plain `fetch` it was measured with.
157
+ */
158
+ pooled = false) {
149
159
  if (config.embeddingProvider === 'none')
150
160
  return null;
151
161
  if (config.embeddingProvider === 'local') {
@@ -166,18 +176,20 @@ vertexSeam) {
166
176
  model: config.embeddingModel,
167
177
  dimensions: config.embeddingDims,
168
178
  ...vertexSeam,
179
+ // An embedding is a pure function of its text, so a request whose idle connection the service closed meanwhile is sent again.
180
+ // A connection it has to open for a read is told to that read's log line (callPhases.ts `connect`).
181
+ ...(pooled ? { request: (0, poolFetch_1.poolFetch)({ retryOnReset: true, onConnect: ms => (0, callPhases_1.recordPhase)('connect', ms) }) } : {}),
169
182
  }))), config, `p-${served}`);
170
183
  }
171
184
  const keys = (config.embeddingApiKey ?? '').split(',').map(k => k.trim()).filter(Boolean);
172
185
  return new openai_1.OpenAIEmbeddingProvider(keys, config.embeddingModel, config.embeddingBaseURL, config.embeddingDims);
173
186
  }
174
187
  function attachEngine(db, config, override) {
175
- const provider = override === undefined ? createEmbeddingProvider(config) : override;
188
+ const provider = override === undefined ? createEmbeddingProvider(config, undefined, true) : override;
176
189
  if (!provider) {
177
190
  // What this process saves gets no vector: an index that has a model's vectors is marked unfinished first.
178
191
  if ((0, vectorIndex_1.markVectorsUnfinished)(db)) {
179
- console.log('[embeddings] no embedding provider is configured: what is saved now gets no vector, so the index is marked unfinished; '
180
- + 'the next start with its model embeds those chunks before semantic search reads the index again');
192
+ log.info('embeddings_unconfigured', { detail: 'no embedding provider is configured: what is saved now gets no vector, so the index is marked unfinished; the next start with its model embeds those chunks before semantic search reads the index again' });
181
193
  }
182
194
  return { provider: null, vectorIndex: null, dims: readEmbeddingTableDims(db) };
183
195
  }
@@ -237,14 +249,14 @@ function claimEmbeddingSpace(db, vectorIndex, provider) {
237
249
  return;
238
250
  }
239
251
  vectorIndex.beginRebuild(wanted); // there is no vector to clear: this names the index and marks it unfinished
240
- console.log(`[embeddings] index.db holds ${chunks} chunk(s) and no vectors (it was written without an embedding provider): `
241
- + `every chunk is embedded with ${wanted} in the background, and until that finishes semantic search is off and keyword search keeps working.`);
252
+ log.info('embeddings_backfill', { chunks, model: wanted, detail: 'index.db holds no vectors (it was written without an embedding provider): every chunk is embedded in the background; until then semantic search is off and keyword search keeps working' });
242
253
  return;
243
254
  }
244
- console.warn(stored === null
245
- ? '[embeddings] index.db holds vectors but no record of the model that wrote them (it was built before models were recorded). '
246
- : `[embeddings] index.db holds vectors from ${stored}, but the configured embedding is ${wanted}. `, `Search cannot use vectors from another model: a rebuild embeds every chunk again with ${wanted}, and until it finishes `
247
- + 'semantic search is off and keyword search keeps working.');
255
+ log.warn('embeddings_model_mismatch', {
256
+ stored: stored ?? 'unrecorded',
257
+ model: wanted,
258
+ detail: 'search cannot use vectors from another model: a rebuild embeds every chunk again, and until it finishes semantic search is off and keyword search keeps working',
259
+ });
248
260
  }
249
261
  /**
250
262
  * Whether a semantic search may read this engine's index: every vector in it is in the configured model's space and
@@ -319,18 +331,18 @@ function engineRebuildEmbeddings(db, engine, options = {}) {
319
331
  };
320
332
  const run = async () => {
321
333
  const chunks = () => db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
322
- console.log(`[embeddings] rebuilding the vectors of ${chunks()} chunk(s) with ${space}; semantic search is off until it finishes`);
334
+ log.info('embedding_rebuild_started', { chunks: chunks(), model: space, detail: 'semantic search is off until it finishes' });
323
335
  for (let outage = 0; !controller.signal.aborted; outage++) {
324
336
  try {
325
337
  const result = await (0, rebuild_1.rebuildEmbeddings)(db, vectorIndex, provider, space, { ...rebuildOptions, signal: controller.signal });
326
338
  if (result.done) {
327
- console.log(`[embeddings] rebuild finished in ${((Date.now() - startedAt) / 1000).toFixed(1)} s; semantic search is on`);
339
+ log.info('embedding_rebuild_finished', { ms: Date.now() - startedAt, detail: 'semantic search is on' });
328
340
  return;
329
341
  }
330
342
  }
331
343
  catch (err) {
332
344
  const delay = outageDelayMs(outage);
333
- console.error(`[embeddings] rebuild interrupted with ${(0, rebuild_1.chunksWithoutVectors)(db)} of ${chunks()} chunk(s) still to embed; semantic search stays off; trying again in ${Math.round(delay / 1000)} s:`, err);
345
+ log.error('embedding_rebuild_interrupted', { unembedded: (0, rebuild_1.chunksWithoutVectors)(db), chunks: chunks(), retryInS: Math.round(delay / 1000), detail: 'semantic search stays off' }, err);
334
346
  await wait(delay);
335
347
  }
336
348
  }
@@ -340,7 +352,7 @@ function engineRebuildEmbeddings(db, engine, options = {}) {
340
352
  const rebuild = {
341
353
  abort: () => controller.abort(),
342
354
  waiting: () => waiting,
343
- done: run().catch((err) => console.error('[embeddings] rebuild failed:', err)).finally(() => embeddingRebuilds.delete(vectorIndex)),
355
+ done: run().catch((err) => log.error('embedding_rebuild_failed', {}, err)).finally(() => embeddingRebuilds.delete(vectorIndex)),
344
356
  };
345
357
  embeddingRebuilds.set(vectorIndex, rebuild);
346
358
  return rebuild;
@@ -386,7 +398,7 @@ function resolveEngineDigests(db, config, engine, idle = null, env = process.env
386
398
  if (!options)
387
399
  return null;
388
400
  if (!engine.provider) {
389
- console.log('[digests] RENOR_DIGESTS is on, but this workspace has no embedding provider, so digests are off for it');
401
+ log.info('digests_off', { detail: 'RENOR_DIGESTS is on, but this workspace has no embedding provider, so digests are off for it' });
390
402
  return null;
391
403
  }
392
404
  const worker = new worker_1.DigestWorker({
@@ -396,7 +408,7 @@ function resolveEngineDigests(db, config, engine, idle = null, env = process.env
396
408
  provider: engine.provider,
397
409
  track: idle ? work => idle.track(work) : undefined,
398
410
  });
399
- console.log(`[digests] write-time digests are on (model ${options.model}, at most ${options.maxUsdPerDay} USD a day)`);
411
+ log.info('digests_on', { model: options.model, maxUsdPerDay: options.maxUsdPerDay });
400
412
  return { options, worker };
401
413
  }
402
414
  /**
@@ -463,7 +475,7 @@ bestFirst = false) {
463
475
  // Search by meaning failed: the embedding provider is down, or refuses the key or the quota (or the semantic lane
464
476
  // broke, which the log shows). The facts and keyword search do not need it, so a caller answers with those and
465
477
  // says why nothing was found by meaning, instead of failing whole.
466
- console.error('[search] semantic search failed:', err);
478
+ log.error('semantic_search_failed', {}, err);
467
479
  semanticFailures.set(engine, { at: new Date().toISOString(), reason: failureReason(err, 120) });
468
480
  return { ok: false, error: 'semantic-unavailable', notice: semanticFailureNotice(err), hits: [] };
469
481
  }
@@ -527,7 +539,7 @@ bestFirst = false) {
527
539
  }
528
540
  catch (err) {
529
541
  // Optional augmentation: a fault here never fails the search.
530
- console.error('[digests] reading the digests failed:', failureReason(err, 200));
542
+ log.error('digests_read_failed', {}, err);
531
543
  }
532
544
  }
533
545
  const isConversation = withChunkIds ? (0, readerLayout_1.conversationNoteMatcher)(db) : () => false;
@@ -556,11 +568,12 @@ bestFirst = false) {
556
568
  ...(scopeNorm ? { scope: scopeNorm } : {}),
557
569
  };
558
570
  }
559
- function engineSearchText(db, memoryDir, query, topK, scope, includeHistory = false, ledgerRelPath = lifecycle_1.DEFAULT_LEDGER_REL, ledgerRecords) {
571
+ function engineSearchText(db, memoryDir, query, topK, scope, includeHistory = false, ledgerRelPath = lifecycle_1.DEFAULT_LEDGER_REL, ledgerRecords, matchOverride) {
560
572
  const fts = (0, statements_1.prepareOnce)(db, `SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'chunks_fts'`).get();
561
573
  if (!fts)
562
574
  return { ok: false, error: 'fts-unavailable', results: [] };
563
- const match = (0, queryHelpers_1.quoteFtsQuery)(query);
575
+ // matchOverride (H11b): an FTS expression built by the caller (an OR over the question's content words); absent, the whole query as one phrase.
576
+ const match = matchOverride ?? (0, queryHelpers_1.quoteFtsQuery)(query);
564
577
  if (!match)
565
578
  return { ok: true, results: [] };
566
579
  const scopeNorm = (0, lifecycle_1.scopeFilterFor)(scope);
@@ -872,7 +885,7 @@ pass = {}) {
872
885
  // Not a failure to retry: it stays out of the index until the file changes, and is said once per change.
873
886
  known.delete(f.file);
874
887
  tooLarge.set(f.file, stampOf(f));
875
- console.warn(`[index] ${f.file}: ${err.message}`);
888
+ log.warn('note_too_large', { file: path_1.default.relative(config.memoryDir, f.file) }, err);
876
889
  return;
877
890
  }
878
891
  known.delete(f.file);
@@ -880,7 +893,7 @@ pass = {}) {
880
893
  return;
881
894
  errors++;
882
895
  failure ??= failureReason(err, 300);
883
- console.error(`[index] error indexing ${f.file}:`, err);
896
+ log.error('index_file_failed', { file: path_1.default.relative(config.memoryDir, f.file) }, err);
884
897
  }
885
898
  });
886
899
  // Settle what was checked: a file whose stamp is recorded (or that is named too large for its stamp) is done; any
@@ -934,7 +947,7 @@ provider, query, asked) {
934
947
  await engineIndexChanged(db, { memoryDir }, engine, undefined, { interactive: true });
935
948
  }
936
949
  catch (err) {
937
- console.error('[index] finishing the unfinished files failed:', err);
950
+ log.error('index_finish_failed', {}, err);
938
951
  }
939
952
  const after = unfinished.get();
940
953
  if (after > 0)
@@ -1007,7 +1020,7 @@ function engineFinishIndexInBackground(db, config, engine, options = {}) {
1007
1020
  await engineIndexChanged(db, config, engine, undefined, { interactive: true });
1008
1021
  if (unfinished.get() === 0 && (0, rebuild_1.chunksWithoutVectors)(db) === 0) {
1009
1022
  if (outage > 0)
1010
- console.log('[index] the embedding service answers again; what was saved while it did not is findable by meaning now');
1023
+ log.info('embeddings_recovered', { detail: 'the embedding service answers again; what was saved while it did not is findable by meaning now' });
1011
1024
  return;
1012
1025
  }
1013
1026
  }
@@ -1020,7 +1033,7 @@ function engineFinishIndexInBackground(db, config, engine, options = {}) {
1020
1033
  if (controller.signal.aborted)
1021
1034
  return;
1022
1035
  const delay = retryDelayMs(outage);
1023
- console.error(`[index] finishing the index in the background failed (${failureReason(err, 200)}); ${(0, rebuild_1.chunksWithoutVectors)(db)} chunk(s) have no vector yet; trying again in ${Math.round(delay / 1000)} s`);
1036
+ log.error('background_index_retry', { unembedded: (0, rebuild_1.chunksWithoutVectors)(db), retryInS: Math.round(delay / 1000) }, err);
1024
1037
  await wait(delay);
1025
1038
  }
1026
1039
  }
@@ -1028,7 +1041,7 @@ function engineFinishIndexInBackground(db, config, engine, options = {}) {
1028
1041
  const job = {
1029
1042
  abort: () => controller.abort(),
1030
1043
  waiting: () => waiting,
1031
- done: run().catch((err) => console.error('[index] background indexing failed:', err)).finally(() => {
1044
+ done: run().catch((err) => log.error('background_index_failed', {}, err)).finally(() => {
1032
1045
  if (backgroundIndexes.get(db) === job)
1033
1046
  backgroundIndexes.delete(db);
1034
1047
  }),
@@ -3,7 +3,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.NOT_EXPORTED = exports.EXPORT_SECTIONS = exports.EXPORT_FORMAT_VERSION = void 0;
6
+ exports.HANDOFF_NOT_EXPORTED = exports.NOT_EXPORTED = exports.EXPORT_SECTIONS = exports.EXPORT_FORMAT_VERSION = void 0;
7
+ exports.notExported = notExported;
7
8
  exports.exportEverything = exportEverything;
8
9
  const promises_1 = __importDefault(require("fs/promises"));
9
10
  const path_1 = __importDefault(require("path"));
@@ -11,6 +12,7 @@ const lifecycle_1 = require("../memory/lifecycle");
11
12
  const dailyNote_1 = require("../memory/dailyNote");
12
13
  const memoryMap_1 = require("../memorymap/memoryMap");
13
14
  const parser_1 = require("../indexer/parser");
15
+ const handoff_1 = require("../memory/handoff");
14
16
  const replacements_1 = require("../memorymap/replacements");
15
17
  // ---------------------------------------------------------------------------
16
18
  // export_memory: everything one workspace holds, in one documented Markdown format ("renor-export/v1").
@@ -41,6 +43,19 @@ exports.NOT_EXPORTED = [
41
43
  { kind: 'searchIndex', what: 'the search index and embeddings', reason: 'derived: index_memory rebuilds them from the exported files' },
42
44
  { kind: 'otherFiles', what: 'files in the memory folder that are not Markdown (.md), dot-folders other than the retired area, and symbolic links', reason: 'the export carries Markdown notes only' },
43
45
  ];
46
+ /**
47
+ * Named in notExported only where RENOR_HANDOFF_NOTE is on: with it off there is no handoff note and no table, and the
48
+ * manifest is the one it was without it.
49
+ */
50
+ exports.HANDOFF_NOT_EXPORTED = {
51
+ kind: 'handoffNotes',
52
+ what: 'the handoff note of each repository and branch (the handoff_notes table)',
53
+ reason: 'session status, not memory: it expires after 7 days and stays out of every read',
54
+ };
55
+ /** NOT_EXPORTED, and the handoff notes where RENOR_HANDOFF_NOTE is on. */
56
+ function notExported(env = process.env) {
57
+ return (0, handoff_1.handoffNoteEnabled)(env) ? [...exports.NOT_EXPORTED, exports.HANDOFF_NOT_EXPORTED] : exports.NOT_EXPORTED;
58
+ }
44
59
  /** Characters per reply: well under the MCP result limits this codebase works to. */
45
60
  const DEFAULT_MAX_CHARS = 600_000;
46
61
  function maxChars() {
@@ -281,7 +296,7 @@ async function exportEverything(ctx, opts) {
281
296
  bytes: built.ledger?.length ?? null,
282
297
  counts,
283
298
  sections,
284
- notExported: exports.NOT_EXPORTED,
299
+ notExported: notExported(),
285
300
  };
286
301
  let document = '';
287
302
  const included = [];
@@ -3,9 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.factAuthorsByEvent = factAuthorsByEvent;
4
4
  exports.eventMarker = eventMarker;
5
5
  exports.markFactHits = markFactHits;
6
+ exports.claimsReservedEntrySource = claimsReservedEntrySource;
6
7
  const statements_1 = require("../db/statements");
7
8
  const sourceName_1 = require("../memory/sourceName");
8
9
  const ledgerLiveness_1 = require("../retrieval/ledgerLiveness");
10
+ const dailyNote_1 = require("../memory/dailyNote");
9
11
  /**
10
12
  * The author of each saved fact among these ledger events, by event id. Ids are lower-case in and out (ledgerEventIds);
11
13
  * an event with no fact is absent from the map, so a memory that is not a fact is never marked.
@@ -19,7 +21,7 @@ function factAuthorsByEvent(db, eventIds) {
19
21
  }
20
22
  /**
21
23
  * The marker of a memory that is one or more ledger events. A chunk normally holds one event; when it holds several
22
- * facts of several authors, none is named, and an import among them is still marked untrusted.
24
+ * facts of several authors, none is named, and an import or a reserved source among them is still marked untrusted.
23
25
  */
24
26
  function eventMarker(eventIds, authors, reader) {
25
27
  const saved = eventIds.filter(id => authors.has(id)).map(id => authors.get(id) ?? '');
@@ -28,7 +30,7 @@ function eventMarker(eventIds, authors, reader) {
28
30
  const named = [...new Set(saved)];
29
31
  if (named.length === 1)
30
32
  return (0, sourceName_1.authorMarker)(named[0], reader);
31
- return saved.some(sourceName_1.isImportSource) ? { untrusted: true } : {};
33
+ return saved.some(author => (0, sourceName_1.isImportSource)(author) || (0, sourceName_1.isReservedSource)(author)) ? { untrusted: true } : {};
32
34
  }
33
35
  /**
34
36
  * Marks, in place, each search hit that is a saved fact: a ledger event by its author, and a fact an identity question
@@ -47,3 +49,17 @@ function markFactHits(db, hits, reader, isLedger) {
47
49
  }
48
50
  }
49
51
  }
52
+ /**
53
+ * Whether a chunk of a daily log holds an entry whose header claims a reserved source (`- [09:00:00] (system) ...`): an entry
54
+ * saved before the reserved-identity rule, or written into the file by hand. The search reads (get_context, search_memory,
55
+ * search_text) mark such a chunk `untrusted`, as memory_map and read_memory mark the entry. A chunk is the unit a reply
56
+ * carries, so the mark is on the whole chunk; text is never changed.
57
+ */
58
+ function claimsReservedEntrySource(notePath, text) {
59
+ if (typeof notePath !== 'string' || typeof text !== 'string' || !dailyNote_1.DAILY_FILE_RE.test(notePath))
60
+ return false;
61
+ return text.split('\n').some(line => {
62
+ const header = (0, dailyNote_1.parseEntryHeader)(line);
63
+ return header !== null && (0, sourceName_1.isReservedSource)(header.source);
64
+ });
65
+ }
@@ -195,17 +195,18 @@ function collectEligible(historyDir) {
195
195
  skipped++;
196
196
  continue;
197
197
  }
198
- if (entry.carried === true) {
198
+ const record = entry;
199
+ if (record.carried === true) {
199
200
  skipped++;
200
201
  continue;
201
202
  } // withdrawn: excluded
202
- const rel = entry.rel;
203
+ const rel = record.rel;
203
204
  if (typeof rel !== 'string' || rel !== `notes/${name}`) {
204
205
  skipped++;
205
206
  continue;
206
207
  }
207
- const expectedSha = entry.note_sha256;
208
- const convId = entry.conversation_id;
208
+ const expectedSha = record.note_sha256;
209
+ const convId = record.conversation_id;
209
210
  if (typeof expectedSha !== 'string' || !/^[0-9a-f]{64}$/.test(expectedSha)) {
210
211
  skipped++;
211
212
  continue;
@@ -0,0 +1,118 @@
1
+ "use strict";
2
+ /**
3
+ * One operator-log shape for the engine, the gateway and the Edge Functions: a single JSON line
4
+ * `{ ts, level, svc, event, ...fields, err? }`, written with console.log (info), console.warn (warn) or console.error (error).
5
+ *
6
+ * Source of truth: engine/src/mvp/log.ts. Byte-identical copies live at
7
+ * engine/apps/gateway/src/shared/log.ts and supabase/functions/_shared/log.ts, because each runtime
8
+ * deploys alone; engine/tests/mvp/structured-log.test.ts holds them equal. No imports, so it compiles
9
+ * as CommonJS (engine), NodeNext ESM (gateway) and Deno (Edge Functions).
10
+ *
11
+ * What it refuses to write: field values that are not scalars, field names that are not plain identifiers,
12
+ * keys that name a credential or private content (see SENSITIVE_KEY), and, inside every string it writes
13
+ * (fields, the error's name, message, code and each stack frame): control and bidi characters, URL
14
+ * queries and userinfo, bearer tokens, JWTs, known key formats, credential-labelled values, long
15
+ * unlabelled keys, emails and absolute paths (see REDACTIONS). Strings are cut at 300 characters, so a
16
+ * stray memory excerpt cannot leak whole.
17
+ */
18
+ Object.defineProperty(exports, "__esModule", { value: true });
19
+ exports.serializeError = serializeError;
20
+ exports.formatLogLine = formatLogLine;
21
+ exports.createLogger = createLogger;
22
+ const MAX_STRING = 300;
23
+ const MAX_FRAMES = 5;
24
+ const RESERVED = new Set(['ts', 'level', 'svc', 'event', 'err']);
25
+ const SENSITIVE_KEY = /secret|token|password|passwd|authorization|cookie|api_?key|service_?role|dsn|credential|^body$|^text$|^content$|^query$|^memory$|^email$/i;
26
+ const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_]{0,63}$/;
27
+ // C0/C1 controls, soft hyphen, zero-width and bidi controls, NEL and the Unicode line/paragraph separators: none may
28
+ // reach a log line, where they forge new lines or hide what an operator reads.
29
+ const CONTROL = /[\u0000-\u001f\u007f-\u009f\u00ad\u061c\u180e\u200b-\u200f\u2028-\u202e\u2060-\u206f\ufeff]/g;
30
+ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
31
+ /** Applied in order to every string written; each step names what it takes out. */
32
+ const REDACTIONS = [
33
+ [CONTROL, ' '],
34
+ // Credentials in a URL: userinfo (`scheme://user:pass@`) and the whole query string.
35
+ [/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#@]+@/gi, '$1[redacted]@'],
36
+ [/\?[^\s#"'<>|]*=[^\s#"'<>|]*/g, '?[redacted]'],
37
+ // Bearer tokens, JWTs and the key formats the services Renor talks to use.
38
+ [/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [redacted]'],
39
+ [/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+/g, '[redacted]'],
40
+ [/\b(?:sk|pk|rk|whsec|re|phc|phx|sbp|ghp|gho|ghs|xox[abp])[_-][A-Za-z0-9_-]{12,}/g, '[redacted]'],
41
+ // Any value labelled as a credential: `api_key=...`, `password: ...`, `x-api-key: ...`.
42
+ [/\b((?:x-)?(?:api[_-]?key|access[_-]?key|client[_-]?secret|secret|token|password|passwd|pwd|auth|authorization|signature|sig|session|cookie|apikey))(\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s,;&"']+)/gi, '$1$2[redacted]'],
43
+ // An unlabelled long random-looking string (letters and digits, 32+): a key nobody named. UUIDs are ids, and stay.
44
+ [/\b[A-Za-z0-9_-]{32,}\b/g, (m) => (UUID.test(m) || !/[0-9]/.test(m) || !/[A-Za-z]/.test(m) ? m : '[redacted]')],
45
+ [/[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+/g, '[email]'],
46
+ // Absolute paths: they name the tenant and the user's own files. A path relative to the memory folder stays.
47
+ [/file:\/\/\/?[^\s()'"`,;]+/g, '[path]'],
48
+ [/(^|[\s(\[='"`,])\/(?:[^\s/()'"`,;]+\/)+[^\s/()'"`,;:]+/g, '$1[path]'],
49
+ [/\b[A-Za-z]:\\[^\s'"`,;]*/g, '[path]'],
50
+ ];
51
+ function clean(value) {
52
+ let out = value;
53
+ for (const [pattern, replacement] of REDACTIONS)
54
+ out = out.replace(pattern, replacement);
55
+ return out.length > MAX_STRING ? `${out.slice(0, MAX_STRING)}…` : out;
56
+ }
57
+ /** One stack frame: its function and the code file's name and position, never the directory it runs from. */
58
+ function cleanFrame(frame) {
59
+ return clean(frame.trim().replace(/(?:file:\/\/)?(?:\/|[A-Za-z]:\\)[^\s()]*?([^/\\\s()]+\.[cm]?[jt]sx?):(\d+):(\d+)/g, '$1:$2:$3'));
60
+ }
61
+ /** The error as it is reported everywhere: name, a cut message, an optional code and the top stack frames, each cleaned. */
62
+ function serializeError(err) {
63
+ if (!(err instanceof Error))
64
+ return { name: 'NonError', message: clean(String(err)) };
65
+ const out = { name: clean(String(err.name)), message: clean(String(err.message)) };
66
+ const code = err.code;
67
+ if (typeof code === 'string' || typeof code === 'number')
68
+ out.code = clean(String(code));
69
+ const frames = String(err.stack ?? '').split('\n').filter((line) => /^\s+at /.test(line)).slice(0, MAX_FRAMES);
70
+ if (frames.length > 0)
71
+ out.stack = frames.map(cleanFrame).join(' | ');
72
+ return out;
73
+ }
74
+ /** The JSON line for one event; exported so a test can read exactly what would be written. */
75
+ function formatLogLine(svc, level, event, fields = {}, err) {
76
+ const line = { ts: new Date().toISOString(), level, svc: clean(svc), event: clean(event) };
77
+ for (const [key, value] of Object.entries(fields)) {
78
+ if (value === undefined || RESERVED.has(key) || !SAFE_KEY.test(key))
79
+ continue;
80
+ if (SENSITIVE_KEY.test(key))
81
+ line[key] = '[redacted]';
82
+ else if (typeof value === 'string')
83
+ line[key] = clean(value);
84
+ else if (typeof value === 'number' || typeof value === 'boolean' || value === null)
85
+ line[key] = value;
86
+ }
87
+ if (err !== undefined)
88
+ line.err = serializeError(err);
89
+ return JSON.stringify(line);
90
+ }
91
+ /**
92
+ * `stderr: true` writes info lines to stderr too, for code that also runs where stdout is data (the stdio MCP server, a
93
+ * CLI that prints JSON). This module is the only place in the engine, the gateway and the Edge Functions' shared code
94
+ * allowed to call console; engine/tests/mvp/no-raw-console.test.ts holds that.
95
+ */
96
+ function createLogger(svc, options = {}) {
97
+ const write = (level, event, fields, err) => {
98
+ try {
99
+ const line = formatLogLine(svc, level, event, fields, err);
100
+ if (level === 'info' && !options.stderr)
101
+ console.log(line);
102
+ else if (level === 'info')
103
+ console.error(line);
104
+ else if (level === 'warn')
105
+ console.warn(line);
106
+ else
107
+ console.error(line);
108
+ }
109
+ catch {
110
+ // Logging must never break the product.
111
+ }
112
+ };
113
+ return {
114
+ info: (event, fields) => write('info', event, fields),
115
+ warn: (event, fields, err) => write('warn', event, fields, err),
116
+ error: (event, fields, err) => write('error', event, fields, err),
117
+ };
118
+ }