renor-engine 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +7 -2
  2. package/dist/.tsbuildinfo +1 -1
  3. package/dist/cli.js +26 -0
  4. package/dist/eval/indexSnapshots.js +1 -0
  5. package/dist/evidence/ollamaChat.js +1 -1
  6. package/dist/extraction/jevDecisionModel.js +307 -0
  7. package/dist/extraction/secretGuard.js +7 -3
  8. package/dist/learning/cli.js +113 -0
  9. package/dist/learning/schema.js +144 -0
  10. package/dist/learning/settings.js +45 -0
  11. package/dist/learning/store.js +267 -0
  12. package/dist/learning/tap.js +401 -0
  13. package/dist/memory/dreamer/adjudicate.js +215 -0
  14. package/dist/memory/dreamer/apply.js +120 -0
  15. package/dist/memory/dreamer/budget.js +64 -0
  16. package/dist/memory/dreamer/candidates.js +149 -0
  17. package/dist/memory/dreamer/collection.js +104 -0
  18. package/dist/memory/dreamer/dreamer.js +352 -0
  19. package/dist/memory/dreamer/review.js +111 -0
  20. package/dist/memory/factConflicts.js +2 -0
  21. package/dist/memory/lifecycle.js +28 -5
  22. package/dist/memory/writer.js +3 -2
  23. package/dist/mvp/agentMemory.js +3 -0
  24. package/dist/mvp/cli.js +11 -0
  25. package/dist/mvp/config.js +4 -0
  26. package/dist/mvp/exportAll.js +19 -0
  27. package/dist/mvp/server.js +118 -10
  28. package/dist/retrieval/orchestrator.js +69 -16
  29. package/dist/retrieval/rankingTrace.js +184 -0
  30. package/dist/sharing/cli.js +181 -0
  31. package/dist/sharing/examples.js +136 -0
  32. package/dist/sharing/fields.js +225 -0
  33. package/dist/sharing/loop.js +80 -0
  34. package/dist/sharing/project.js +327 -0
  35. package/dist/sharing/send.js +466 -0
  36. package/dist/sharing/state.js +222 -0
  37. package/dist/sharing/wording.js +48 -0
  38. package/package.json +1 -1
  39. package/dist/BUILD.json +0 -6
@@ -14,6 +14,8 @@ const memoryMap_1 = require("../memorymap/memoryMap");
14
14
  const parser_1 = require("../indexer/parser");
15
15
  const handoff_1 = require("../memory/handoff");
16
16
  const replacements_1 = require("../memorymap/replacements");
17
+ const settings_1 = require("../learning/settings");
18
+ const store_1 = require("../learning/store");
17
19
  // ---------------------------------------------------------------------------
18
20
  // export_memory: everything one workspace holds, in one documented Markdown format ("renor-export/v1").
19
21
  //
@@ -41,6 +43,7 @@ exports.NOT_EXPORTED = [
41
43
  { kind: 'agentState', what: 'connected agent names, read cursors and live leases (known_agents, agent_cursors, agent_leases)', reason: 'connection state, not memory' },
42
44
  { kind: 'searchHistory', what: 'the promoted history directory that search_history reads (MVP_HISTORY_DIR)', reason: 'configured outside the memory folder' },
43
45
  { kind: 'searchIndex', what: 'the search index and embeddings', reason: 'derived: index_memory rebuilds them from the exported files' },
46
+ { kind: 'sharingStore', what: 'the sharing choice, sender state and optional text examples in .memory-index/sharing/', reason: 'consent state and sender credentials are local controls, not portable memory; the learningLog manifest field separately exports learning.sqlite' },
44
47
  { kind: 'otherFiles', what: 'files in the memory folder that are not Markdown (.md), dot-folders other than the retired area, and symbolic links', reason: 'the export carries Markdown notes only' },
45
48
  ];
46
49
  /**
@@ -267,6 +270,19 @@ async function buildSections(ctx, wanted) {
267
270
  }
268
271
  return built;
269
272
  }
273
+ /** The portable contents of learning.sqlite, when that optional log exists. */
274
+ function exportLearningLog(memoryDir) {
275
+ const file = (0, settings_1.learningDbPath)(memoryDir);
276
+ const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(), rawQueries: false, create: false });
277
+ if (log === null)
278
+ return null;
279
+ try {
280
+ return { file: '.memory-index/learning.sqlite', data: log.exportLog() };
281
+ }
282
+ finally {
283
+ log.close();
284
+ }
285
+ }
270
286
  function sectionHeader(name, count, from) {
271
287
  return `\n# ${exports.EXPORT_FORMAT_VERSION} · ${name} (${count} item${count === 1 ? '' : 's'}${from > 0 ? `, continuing from item ${from}` : ''})\n\n`;
272
288
  }
@@ -297,6 +313,9 @@ async function exportEverything(ctx, opts) {
297
313
  counts,
298
314
  sections,
299
315
  notExported: notExported(),
316
+ // learning.sqlite is user-owned workspace data, not a rebuildable search index. Export its
317
+ // portable rows (never its private HMAC secret) alongside the Markdown memory document.
318
+ learningLog: exportLearningLog(ctx.memoryDir),
300
319
  };
301
320
  let document = '';
302
321
  const included = [];
@@ -57,6 +57,12 @@ const questionSpan_1 = require("../retrieval/questionSpan");
57
57
  const readerEvents_1 = require("../retrieval/readerEvents");
58
58
  const orchestrator_1 = require("../retrieval/orchestrator");
59
59
  const buildIdentity_1 = require("../buildIdentity");
60
+ const tap_1 = require("../learning/tap");
61
+ const rankingTrace_1 = require("../retrieval/rankingTrace");
62
+ const examples_1 = require("../sharing/examples");
63
+ const fields_1 = require("../sharing/fields");
64
+ const send_1 = require("../sharing/send");
65
+ const state_1 = require("../sharing/state");
60
66
  const hostHeaderValidation_js_1 = require("@modelcontextprotocol/sdk/server/middleware/hostHeaderValidation.js");
61
67
  const apiAuth_1 = require("../api/apiAuth");
62
68
  const memoryDir_1 = require("./memoryDir");
@@ -64,6 +70,7 @@ const hosted_1 = require("../importing/hosted");
64
70
  const exportAll_1 = require("./exportAll");
65
71
  const memoryMap_1 = require("../memorymap/memoryMap");
66
72
  const idle_1 = require("./idle");
73
+ const dreamer_1 = require("../memory/dreamer/dreamer");
67
74
  const activity_1 = require("./activity");
68
75
  const storedSize_1 = require("./storedSize");
69
76
  const sanitize_1 = require("./sanitize");
@@ -456,11 +463,12 @@ function withAnnotationTitle(config) {
456
463
  return { ...config, annotations: { ...config.annotations, title: config.title } };
457
464
  }
458
465
  /**
459
- * The one tool only the dashboard may list or run: it erases retired memories for good. Its registerTool call below spells the name
460
- * as a literal, because the docs binding and the hosted-tool tests read the registry off the literal call sites; if the two ever
461
- * differ, registeredToolEntry throws when an agent's server is built.
466
+ * The tools only the dashboard may list or run: one erases retired memories for good, the other reads and sets the workspace's
467
+ * choice to share learning data. Each registerTool call below spells its name as a literal, because the docs binding and the
468
+ * hosted-tool tests read the registry off the literal call sites; if the two ever differ, registeredToolEntry throws when an
469
+ * agent's server is built.
462
470
  */
463
- const OWNER_ONLY_TOOL = 'delete_retired_memory';
471
+ const OWNER_ONLY_TOOLS = ['delete_retired_memory', 'sharing_settings'];
464
472
  /** The tools agents use to reach each other. A directory client is offered none of them; registeredToolEntry throws if one is renamed. */
465
473
  const HIVE_TOOLS = ['whats_new', 'ask_agent', 'list_requests', 'answer_request', 'close_request'];
466
474
  /**
@@ -478,7 +486,7 @@ function registeredToolEntry(server, name) {
478
486
  * binds one transport per Protocol instance, so SSE sessions each get
479
487
  * their own server (bounded, in-process); JSON mode shares one.
480
488
  */
481
- function buildMcpServer(db, config, engine, idle, digests, usefulness, connection, announceFirst, telemetrySession = null, dashboard = false, directory = false) {
489
+ function buildMcpServer(db, config, engine, idle, digests, usefulness, connection, announceFirst, telemetrySession = null, dashboard = false, directory = false, dreamer = null) {
482
490
  // RENOR_TOOL_OUTPUT_SCHEMAS, read once for this server: its tools list an outputSchema exactly when its answers carry
483
491
  // structuredContent.
484
492
  const withSchemas = toolOutputSchemasEnabled();
@@ -531,6 +539,16 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
531
539
  function toolResultOk(result) {
532
540
  return !(result && result.isError);
533
541
  }
542
+ // The learning log (RENOR_LEARNING_LOG, src/learning): null unless it is on, and then one record per successful call of the
543
+ // tools it covers. The observer reads the reply before it is decorated and never changes it.
544
+ const learner = (0, tap_1.learningObserver)(config.learning, {
545
+ actor: dashboard ? 'dashboard' : 'agent',
546
+ session: telemetrySession,
547
+ model: () => engine.modelKey ?? null,
548
+ agent: extra => agentOf(extra, config),
549
+ // The owner's choice to share, read at each call: recording turns on when they choose a level and off when they choose off.
550
+ sharing: config.sharingDir === undefined ? undefined : (0, examples_1.sharingHook)(config.sharingDir),
551
+ });
534
552
  /** Privacy-safe: tool name / ok / ms and, on a refusal, its code from the closed set — never args or memory content. */
535
553
  function tracedTool(tool, handler) {
536
554
  // On-demand: a running tool call (a write, an index job) holds the engine awake
@@ -547,8 +565,12 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
547
565
  return report && Object.keys(report).length > 0 ? { phases: report } : {};
548
566
  };
549
567
  try {
550
- const result = await (phases ? phases.run(() => run(...args)) : run(...args));
568
+ // The ranker's terms are collected only for the learning log (rankingTrace.ts): with the log off no scope is opened.
569
+ const trace = learner ? new rankingTrace_1.RankingTrace(config.memoryDir) : undefined;
570
+ const invoke = () => (phases ? phases.run(() => run(...args)) : run(...args));
571
+ const result = await (trace ? (0, rankingTrace_1.withRankingTrace)(trace, invoke) : invoke());
551
572
  const ok = toolResultOk(result);
573
+ learner?.observe(tool, args[0], result, ok, Date.now() - started, args[1], trace);
552
574
  (0, telemetry_1.emitTelemetry)({
553
575
  event: 'tool_use',
554
576
  tool,
@@ -687,6 +709,7 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
687
709
  // FULL_VAULT_CHECK_MS (see engineIndexChanged).
688
710
  let refresh = {};
689
711
  if (receipt.action !== 'nochange') {
712
+ dreamer?.noteFactSave();
690
713
  try {
691
714
  const pass = await (0, engine_1.engineIndexChanged)(db, config, engine, undefined, {
692
715
  interactive: true,
@@ -854,6 +877,8 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
854
877
  duplicates = (0, factConflicts_1.possibleDuplicates)(db, receipt, receipt.created?.factId ?? null, (0, lifecycle_1.scopeFilterFor)(args.scope), readerOf(extra));
855
878
  }
856
879
  catch { /* no hint */ }
880
+ if (receipt.action !== 'nochange')
881
+ dreamer?.noteFactSave();
857
882
  return jsonResult({
858
883
  ok: receipt.ok,
859
884
  action: receipt.action,
@@ -981,6 +1006,7 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
981
1006
  counts: zod_1.z.record(zod_1.z.string(), zod_1.z.unknown()),
982
1007
  sections: zod_1.z.array(zod_1.z.object({ name: zod_1.z.string(), items: zod_1.z.number().nullable(), events: zod_1.z.number().nullable().optional() })),
983
1008
  notExported: zod_1.z.array(zod_1.z.object({ kind: zod_1.z.string(), what: zod_1.z.string(), reason: zod_1.z.string() })),
1009
+ learningLog: zod_1.z.union([zod_1.z.null(), zod_1.z.object({ file: zod_1.z.string(), data: zod_1.z.record(zod_1.z.string(), zod_1.z.unknown()) })]),
984
1010
  included: zod_1.z.array(zod_1.z.string()),
985
1011
  truncated: zod_1.z.boolean(),
986
1012
  nextSection: zod_1.z.string().nullable(),
@@ -1573,7 +1599,76 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
1573
1599
  }
1574
1600
  });
1575
1601
  if (!dashboard)
1576
- registeredToolEntry(server, OWNER_ONLY_TOOL).enabled = false;
1602
+ registeredToolEntry(server, 'delete_retired_memory').enabled = false;
1603
+ // Only for the owner, from the dashboard: the workspace's choice to share learning data with Renor (src/sharing). The consent
1604
+ // record the owner's click writes in the control plane (set_learning_consent) is the proof; THIS is what the tenant obeys, so the
1605
+ // dashboard calls it right after the record is written, and recording and sending start or stop on the next call. It guards
1606
+ // itself exactly as delete_retired_memory does, and for an agent it is neither listed nor callable.
1607
+ server.registerTool('sharing_settings', {
1608
+ title: 'Learning-data sharing choice (owner only)',
1609
+ annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true },
1610
+ description: 'Reads, sets or deletes this workspace\'s choice to share learning data with Renor. action "get" returns the choice; "set" needs level (off, signals or data) and, for signals or data, adult: true and the notice_version the owner saw; "delete" asks Renor to delete what it holds for this workspace and removes the local sharing files. Off is the default.',
1611
+ inputSchema: {
1612
+ action: zod_1.z.enum(['get', 'set', 'delete']).describe('get, set or delete'),
1613
+ level: zod_1.z.enum(fields_1.SHARING_LEVELS).optional().describe('With set: off, signals or data'),
1614
+ notice_version: zod_1.z.string().min(1).max(80).optional().describe('With set: the id of the words the owner saw'),
1615
+ adult: zod_1.z.boolean().optional().describe('With set to signals or data: the owner confirmed they are 18 or older'),
1616
+ },
1617
+ outputSchema: zod_1.z.looseObject({
1618
+ ok: zod_1.z.boolean(),
1619
+ action: zod_1.z.enum(['get', 'set', 'delete']),
1620
+ level: zod_1.z.enum(fields_1.SHARING_LEVELS),
1621
+ effective: zod_1.z.enum(fields_1.SHARING_LEVELS),
1622
+ changedAt: zod_1.z.string().nullable(),
1623
+ noticeVersion: zod_1.z.string().nullable(),
1624
+ remoteDeleted: zod_1.z.boolean().nullable(),
1625
+ contributorId: zod_1.z.string().nullable(),
1626
+ }),
1627
+ }, async (args, extra) => {
1628
+ try {
1629
+ if (!isDashboardCall(extra?.requestInfo?.headers, config)) {
1630
+ throw (0, toolRefusal_1.refusal)('invalid_argument', 'sharing_settings is only available to the owner, from the Renor dashboard');
1631
+ }
1632
+ const dir = config.sharingDir;
1633
+ if (dir === undefined)
1634
+ throw (0, toolRefusal_1.refusal)('invalid_argument', 'sharing is not configured on this server');
1635
+ let remoteDeleted = null;
1636
+ if (args.action === 'set') {
1637
+ if (args.level === undefined)
1638
+ throw (0, toolRefusal_1.refusal)('invalid_argument', 'set needs level');
1639
+ if (args.level !== 'off' && (args.adult !== true || args.notice_version === undefined)) {
1640
+ throw (0, toolRefusal_1.refusal)('invalid_argument', 'sharing signals or data needs adult: true and the notice_version the owner saw');
1641
+ }
1642
+ const result = await (0, send_1.chooseLevel)({
1643
+ dir, level: args.level, wording: args.notice_version ?? 'dashboard', adult: args.adult === true, source: 'dashboard', hosted: true,
1644
+ });
1645
+ remoteDeleted = result.remoteDeleted;
1646
+ }
1647
+ else if (args.action === 'delete') {
1648
+ const result = await (0, send_1.deleteAllShared)({ dir, hosted: true });
1649
+ remoteDeleted = result.remote.requested ? result.remote.ok : null;
1650
+ }
1651
+ const record = (0, state_1.readConsent)(dir);
1652
+ // The name the control plane ties to this workspace, so the endpoint can check the control plane's consent record before it
1653
+ // stores a row. It is the hash of the sender's secret; made on first use only for a workspace that has chosen to share.
1654
+ const sharing = record !== null && record.level !== 'off' && record.adult;
1655
+ return jsonResult({
1656
+ ok: true,
1657
+ action: args.action,
1658
+ contributorId: sharing ? (0, state_1.contributorId)((0, state_1.ensureState)(dir).secret) : null,
1659
+ level: record?.level ?? 'off',
1660
+ effective: (0, state_1.effectiveLevel)(dir),
1661
+ changedAt: record?.changedAt ?? null,
1662
+ noticeVersion: record?.wording ?? null,
1663
+ remoteDeleted,
1664
+ });
1665
+ }
1666
+ catch (err) {
1667
+ return toolError(err);
1668
+ }
1669
+ });
1670
+ if (!dashboard)
1671
+ registeredToolEntry(server, 'sharing_settings').enabled = false;
1577
1672
  server.registerTool('whats_new', {
1578
1673
  title: "What's new for me",
1579
1674
  // It moves this connection's mark, so it is a writing tool, not read-only: a second call returns less.
@@ -1810,7 +1905,8 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
1810
1905
  const forDirectory = directory || isDirectoryCall(headers, config);
1811
1906
  const hidden = new Set();
1812
1907
  if (!isDashboardCall(headers, config))
1813
- hidden.add(OWNER_ONLY_TOOL);
1908
+ for (const name of OWNER_ONLY_TOOLS)
1909
+ hidden.add(name);
1814
1910
  if (forDirectory)
1815
1911
  for (const name of [...HIVE_TOOLS, ...directorySurface_1.DIRECTORY_HIDDEN_TOOLS])
1816
1912
  hidden.add(name);
@@ -1895,7 +1991,15 @@ function createMvpApp(config, engineOverride) {
1895
1991
  digests?.worker.start();
1896
1992
  const usefulness = new readUsefulness_1.ReadUsefulness();
1897
1993
  const connection = (0, connectionStatus_1.connectionStatusFromEnv)(process.env);
1898
- const server = buildMcpServer(db, config, engine, activity, digests, usefulness, connection, false);
1994
+ // The consolidation pass (RENOR_CONSOLIDATE, default off): null, and nothing else changes, unless it is switched on.
1995
+ const dreamer = (0, dreamer_1.createDreamer)(process.env, {
1996
+ db,
1997
+ vaultPath: config.memoryDir,
1998
+ ledgerRelPath: config.ledgerRelPath,
1999
+ index: () => (0, engine_1.engineIndexChanged)(db, config, engine),
2000
+ hold: run => { activity.track(() => run).catch(() => undefined); },
2001
+ });
2002
+ const server = buildMcpServer(db, config, engine, activity, digests, usefulness, connection, false, null, false, false, dreamer);
1899
2003
  // An index an earlier build wrote holds the engine's own daily-note headers as facts, and a note that has not changed
1900
2004
  // is never rebuilt, so they are removed here, before anything is read. A failure must not keep a workspace from serving.
1901
2005
  try {
@@ -2312,7 +2416,7 @@ function createMvpApp(config, engineOverride) {
2312
2416
  // calls, isPlatformInitialize) gets no id: it still counts as a connection, but it writes no first call and its mcp_init
2313
2417
  // carries no id for the recall share to join, so the share counts agents only.
2314
2418
  const telemetrySession = (0, telemetry_1.isPlatformInitialize)(req.body) ? null : crypto_1.default.randomBytes(8).toString('hex');
2315
- const sessionServer = buildMcpServer(db, config, engine, activity, digests, usefulness, connection, true, telemetrySession, isDashboardCall(req.headers, config), isDirectoryCall(req.headers, config));
2419
+ const sessionServer = buildMcpServer(db, config, engine, activity, digests, usefulness, connection, true, telemetrySession, isDashboardCall(req.headers, config), isDirectoryCall(req.headers, config), dreamer);
2316
2420
  transport.onclose = () => {
2317
2421
  sseSessions.delete(sessionId);
2318
2422
  (0, telemetry_1.emitTelemetry)({ event: 'session', kind: 'mcp_close' });
@@ -2404,9 +2508,12 @@ function createMvpApp(config, engineOverride) {
2404
2508
  idle,
2405
2509
  digests,
2406
2510
  reindexPending: pendingPass,
2511
+ dreamer,
2407
2512
  close: () => {
2408
2513
  closing ??= (async () => {
2409
2514
  // First: it is background work, and what it has started it finishes before the database closes.
2515
+ dreamer?.stop();
2516
+ await dreamer?.inFlight()?.catch(() => undefined);
2410
2517
  (0, engine_1.runningEmbeddingRebuild)(engine)?.abort();
2411
2518
  (0, engine_1.runningBackgroundIndex)(db)?.abort();
2412
2519
  // The digest worker uses the database too: its sweep in flight ends before the database closes.
@@ -2557,6 +2664,7 @@ async function startMvpService(config, engineOverride) {
2557
2664
  // Restart recovery: replay the authoritative ledger before readiness.
2558
2665
  // Start-up is the one place a torn final event is set aside: nothing else is appending to this ledger yet.
2559
2666
  await (0, lifecycle_1.replayLedger)(mvp.db, config.memoryDir, config.ledgerRelPath, { setTornTailAside: true });
2667
+ mvp.dreamer?.start();
2560
2668
  // The notes of a folder that moved were removed from the index with their old paths: they are indexed at their
2561
2669
  // current paths before the service listens, so the first answer has them. The mark is cleared only by a pass with no
2562
2670
  // failed file. A file whose embedding the provider still refuses after its retries counts as failed: it keeps its
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ENTITY_LANE_RESERVED = exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
3
+ exports.ENTITY_LANE_RESERVED = exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
4
4
  exports.searchBestFirstEnabled = searchBestFirstEnabled;
5
5
  exports.parseQueryDateRef = parseQueryDateRef;
6
6
  exports.queryDateRefMs = queryDateRefMs;
@@ -41,6 +41,7 @@ const keyword_1 = require("./keyword");
41
41
  const queryHelpers_1 = require("./queryHelpers");
42
42
  const ledgerPresentation_2 = require("./ledgerPresentation");
43
43
  const plainWording_1 = require("./plainWording");
44
+ const rankingTrace_1 = require("./rankingTrace");
44
45
  function envFlagOn(name) {
45
46
  const v = (process.env[name] ?? '').trim().toLowerCase();
46
47
  return v === '1' || v === 'true' || v === 'yes' || v === 'on';
@@ -610,20 +611,31 @@ function resolveRankedExpansion(env = process.env) {
610
611
  };
611
612
  }
612
613
  /**
613
- * RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS (default unset, which is off): the longest conversation statement, in tokens,
614
- * that the RANKED expansion alone may add. The former expansion keeps RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120), and an
615
- * unset, empty, non-numeric or below-1 value changes nothing anywhere, byte for byte. Set, it wins over
616
- * RENOR_SESSION_EXPAND_STATEMENT_TOKENS in the ranked expansion only.
614
+ * The longest conversation statement the RANKED expansion adds when nothing says otherwise: 400 tokens. The former expansion
615
+ * (a command line that does not declare a calibrated model) keeps RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120).
616
+ */
617
+ exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = 400;
618
+ /**
619
+ * RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS (default 400 since the flip of 2026-10-25): the longest conversation statement,
620
+ * in tokens, that the RANKED expansion alone may add. Unset or empty: 400, unless RENOR_SESSION_EXPAND_STATEMENT_TOKENS is set,
621
+ * which an operator's explicit value keeps winning as it always did. A number of 1 or more: that limit, over the general one.
622
+ * The off switch: `0`, a negative number or anything that is not a number returns null, and the ranked expansion then takes
623
+ * RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120 unset), as it did before the flip. The former expansion and the documents are
624
+ * never touched by it.
617
625
  *
618
- * Why it exists: the session expansion skips a conversation statement over its limit, whole, and the long user turn that
619
- * answers a question is often just over 120 tokens (501 to 700 characters: missing from the page 5 times in 21, against 6 in
620
- * 777 for shorter ones, on the Haiku default run of 2026-10-23). The chunker splits a section of over 400 tokens into
621
- * paragraphs, which this limit never takes, so 400 is the largest value that changes anything. Measured as 400 beside
622
- * RENOR_EMBEDDING_SERVED_BY=vertex (the ranked expansion a hosted workspace runs) on the official LongMemEval-S instrument:
623
- * engine/docs/evals/results/ranked-statement-limit-20261024/ and LEDGER section BE. A research switch until a default flip lands.
626
+ * Why 400: the session expansion skips a conversation statement over its limit, whole, and the long user turn that answers a
627
+ * question is often just over 120 tokens (501 to 700 characters: missing from the page 5 times in 21, against 6 in 777 for
628
+ * shorter ones, on the Haiku default run of 2026-10-23). The chunker leaves no section over 400 tokens as one chunk, so 400 is
629
+ * the largest value that changes anything. On the official LongMemEval-S instrument with a Haiku 5.5 reader the ranked
630
+ * expansion with 400 scored 433 of 500 and with 120 (the control) 431, against 418, 417 and 423 for the command-line default;
631
+ * the four pages only the limit completes are right 4 of 4 with it and 0 of 4 without, and the page is 5.5% smaller:
632
+ * engine/docs/evals/results/ranked-statement-limit-20261024/ and LEDGER sections BE and BF.
624
633
  */
625
634
  function rankedStatementTokens(env = process.env) {
626
- const value = Number((env.RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS ?? '').trim());
635
+ const raw = (env.RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS ?? '').trim();
636
+ if (raw === '')
637
+ return (env.RENOR_SESSION_EXPAND_STATEMENT_TOKENS ?? '').trim() === '' ? exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS : null;
638
+ const value = Number(raw);
627
639
  return Number.isFinite(value) && value >= 1 ? Math.floor(value) : null;
628
640
  }
629
641
  /**
@@ -856,6 +868,8 @@ function reserveLanePlaces(ranked, topK, laneTerms) {
856
868
  /** The window of hits of a read: scored, with the graph of its notes, their facts and dates, live and capped per note. Waits for nothing. */
857
869
  function rankWindow(read) {
858
870
  const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap } = read;
871
+ // Set only while the learning log collects the ranker's terms (rankingTrace.ts); otherwise every use below is skipped.
872
+ const trace = (0, rankingTrace_1.currentRankingTrace)();
859
873
  // Raw cosines, kept only for a caller that asked to see them (get_context's relevance signal).
860
874
  const cosineByChunk = opts.probe ? new Map(semanticHits.map(h => [h.chunkId, h.score])) : null;
861
875
  const entityHit = (0, entityLookup_1.lookupEntity)(db, query.trim());
@@ -877,8 +891,11 @@ function rankWindow(read) {
877
891
  const scored = semanticHits.map(h => {
878
892
  let score = h.score;
879
893
  let reason = 'semantic';
894
+ const terms = trace ? (0, rankingTrace_1.newRankingTerms)() : null;
880
895
  if (isSpeakerStub(h.text)) {
881
896
  score -= 0.5;
897
+ if (terms)
898
+ terms[(0, rankingTrace_1.rankingTermIndex)('stub')] -= 0.5;
882
899
  reason = 'semantic+stub';
883
900
  }
884
901
  else {
@@ -886,21 +903,30 @@ function rankWindow(read) {
886
903
  if (boost > 0) {
887
904
  score += boost;
888
905
  reason += '+content';
906
+ if (terms)
907
+ terms[(0, rankingTrace_1.rankingTermIndex)('content')] += boost;
889
908
  }
890
909
  }
891
910
  if (priorityPaths.some(p => h.notePath.includes('/' + p) || h.notePath.startsWith(p))) {
892
911
  score += 0.1;
893
912
  reason += '+priority';
913
+ if (terms)
914
+ terms[(0, rankingTrace_1.rankingTermIndex)('priority')] += 0.1;
894
915
  }
895
916
  if (entityHit?.sourceNote === h.notePath) {
896
917
  score += 0.2;
897
918
  reason += '+entity';
919
+ if (terms)
920
+ terms[(0, rankingTrace_1.rankingTermIndex)('entity')] += 0.2;
898
921
  }
899
922
  if (laneTerms.length > 0) {
900
923
  const share = (0, plainWording_1.laneTermShare)(h.text, laneTerms);
901
924
  if (share > 0) {
902
- score += plainWording_1.ENTITY_LANE_BOOST * share;
925
+ const boost = plainWording_1.ENTITY_LANE_BOOST * share;
926
+ score += boost;
903
927
  reason += '+entity-lane';
928
+ if (terms)
929
+ terms[(0, rankingTrace_1.rankingTermIndex)('entity_lane')] += boost;
904
930
  }
905
931
  }
906
932
  // Time-decay recency: +0.1 for today, linearly decaying to 0 over 30 days.
@@ -914,9 +940,12 @@ function rankWindow(read) {
914
940
  if (recencyBoost > 0.001) {
915
941
  score += recencyBoost;
916
942
  reason += '+recency';
943
+ if (terms)
944
+ terms[(0, rankingTrace_1.rankingTermIndex)('recency')] += recencyBoost;
917
945
  }
918
946
  }
919
- return { ...h, score, reason, session_date };
947
+ const out = { ...h, score, reason, session_date };
948
+ return terms ? (0, rankingTrace_1.attachRankingTerms)(out, terms) : out;
920
949
  });
921
950
  // Graph expansion
922
951
  const hitNotePaths = [...new Set(semanticHits.map(h => h.notePath))];
@@ -1017,6 +1046,8 @@ function rankWindow(read) {
1017
1046
  (f.predicate ?? '').toLowerCase().includes(w)));
1018
1047
  if (factMatch) {
1019
1048
  hit.score += 0.15;
1049
+ if (trace)
1050
+ (0, rankingTrace_1.addRankingTerm)(hit, 'fact', 0.15);
1020
1051
  hit.reason += '+fact';
1021
1052
  }
1022
1053
  // As-of lane: the query names a past date, so hits carrying the same
@@ -1027,11 +1058,15 @@ function rankWindow(read) {
1027
1058
  const lowerText = hit.text.toLowerCase();
1028
1059
  if (lowerText.includes(yearToken)) {
1029
1060
  hit.score += 0.08;
1061
+ if (trace)
1062
+ (0, rankingTrace_1.addRankingTerm)(hit, 'asof_year', 0.08);
1030
1063
  hit.reason += '+asof-year';
1031
1064
  if (asofRef.month !== undefined) {
1032
1065
  const monthName = Object.keys(ASOF_MONTHS).find(m => ASOF_MONTHS[m] === asofRef.month);
1033
1066
  if (monthName && lowerText.includes(monthName)) {
1034
1067
  hit.score += 0.04;
1068
+ if (trace)
1069
+ (0, rankingTrace_1.addRankingTerm)(hit, 'asof_month', 0.04);
1035
1070
  hit.reason += '+asof-month';
1036
1071
  }
1037
1072
  }
@@ -1054,6 +1089,8 @@ function rankWindow(read) {
1054
1089
  // Must clear recency (+0.1 max) + fact boost (+0.15) the current note
1055
1090
  // typically carries, or past-intent queries can never win.
1056
1091
  hit.score += 0.3;
1092
+ if (trace)
1093
+ (0, rankingTrace_1.addRankingTerm)(hit, 'historical_intent', 0.3);
1057
1094
  hit.reason += '+historical-intent';
1058
1095
  }
1059
1096
  }
@@ -1066,10 +1103,14 @@ function rankWindow(read) {
1066
1103
  if (stale.length >= 3 && text.includes(stale)) {
1067
1104
  if (pastIntent) {
1068
1105
  hit.score += 0.1;
1106
+ if (trace)
1107
+ (0, rankingTrace_1.addRankingTerm)(hit, 'past_fact', 0.1);
1069
1108
  hit.reason += '+historical-intent';
1070
1109
  }
1071
1110
  else {
1072
1111
  hit.score -= 0.15;
1112
+ if (trace)
1113
+ (0, rankingTrace_1.addRankingTerm)(hit, 'stale_fact', -0.15);
1073
1114
  hit.reason += '+stale-fact';
1074
1115
  }
1075
1116
  break; // one demotion per hit; stale signal already applied
@@ -1086,6 +1127,8 @@ function rankWindow(read) {
1086
1127
  const cur = (f.object_text ?? '').trim().toLowerCase();
1087
1128
  if (cur.length >= 3 && text.includes(cur)) {
1088
1129
  hit.score += 0.1;
1130
+ if (trace)
1131
+ (0, rankingTrace_1.addRankingTerm)(hit, 'current_fact', 0.1);
1089
1132
  hit.reason += '+current-fact';
1090
1133
  break;
1091
1134
  }
@@ -1094,6 +1137,8 @@ function rankWindow(read) {
1094
1137
  }
1095
1138
  }
1096
1139
  const ranked = dedupeHits(scored.sort(semantic_1.byScoreThenPlace));
1140
+ if (trace)
1141
+ ranked.forEach((hit, index) => trace.pool(hit, index + 1));
1097
1142
  // Fail-open floor. The stub filter must never empty the window: a zero-hit
1098
1143
  // retrieval is a hard abort on the product path, because the benchmark client
1099
1144
  // refuses to score a case with no hits ("Retrieval precondition failed: public
@@ -1112,7 +1157,12 @@ function rankWindow(read) {
1112
1157
  const mixed = (0, ledgerPresentation_1.dropLedgerPreamble)((0, ledgerLiveness_1.resolveLedgerHits)([...datedFactHits, ...contentful.filter(h => !seen.has(h.text))], (0, ledgerLiveness_1.closedLedgerEvents)(db), opts.includeHistory === true), opts.isLedgerNote);
1113
1158
  const diversityCap = resolveSessionDiversityCap(process.env, opts.noteCapWhenUnset);
1114
1159
  // RENOR_NEAR_DUP_CAP (default off): near-copies are capped before the per-note cap, on the visible text.
1115
- let hits = reserveLanePlaces(applySessionDiversity((0, plainWording_1.applyNearDupCap)(mixed, nearDupCap, contentfulText), diversityCap, topK), topK, laneTerms).map(h => ({
1160
+ const windowed = reserveLanePlaces(applySessionDiversity((0, plainWording_1.applyNearDupCap)(mixed, nearDupCap, contentfulText), diversityCap, topK), topK, laneTerms);
1161
+ // The window's candidates are recorded before they are copied into the plain hits below, which carry no terms.
1162
+ if (trace)
1163
+ for (const hit of windowed)
1164
+ trace.touch(hit);
1165
+ let hits = windowed.map(h => ({
1116
1166
  notePath: h.notePath,
1117
1167
  chunkId: h.chunkId,
1118
1168
  headingPath: h.headingPath,
@@ -1167,5 +1217,8 @@ async function finishContext(read) {
1167
1217
  const newestFirst = !bestFirst && newestFirstEnabled() && !(temporal && pastIntent) && !asofPast;
1168
1218
  const rankedBy = newestFirst ? 'session_date' : 'score';
1169
1219
  contentfulHeld.clear(); // nothing outlives the read it was worked out for
1170
- return { query, hits: orderHits(hits, rankedBy), rankedBy };
1220
+ // The order the reply presents its hits in, so a candidate's place in it is recorded for the learning log (rankingTrace.ts).
1221
+ const presented = orderHits(hits, rankedBy);
1222
+ (0, rankingTrace_1.currentRankingTrace)()?.presented(presented);
1223
+ return { query, hits: presented, rankedBy };
1171
1224
  }