renor-engine 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli.js +26 -0
- package/dist/eval/indexSnapshots.js +1 -0
- package/dist/evidence/ollamaChat.js +1 -1
- package/dist/extraction/jevDecisionModel.js +307 -0
- package/dist/extraction/secretGuard.js +7 -3
- package/dist/learning/cli.js +113 -0
- package/dist/learning/schema.js +144 -0
- package/dist/learning/settings.js +45 -0
- package/dist/learning/store.js +267 -0
- package/dist/learning/tap.js +401 -0
- package/dist/memory/dreamer/adjudicate.js +215 -0
- package/dist/memory/dreamer/apply.js +120 -0
- package/dist/memory/dreamer/budget.js +64 -0
- package/dist/memory/dreamer/candidates.js +149 -0
- package/dist/memory/dreamer/collection.js +104 -0
- package/dist/memory/dreamer/dreamer.js +352 -0
- package/dist/memory/dreamer/review.js +111 -0
- package/dist/memory/factConflicts.js +2 -0
- package/dist/memory/lifecycle.js +28 -5
- package/dist/memory/writer.js +3 -2
- package/dist/mvp/agentMemory.js +3 -0
- package/dist/mvp/cli.js +11 -0
- package/dist/mvp/config.js +4 -0
- package/dist/mvp/exportAll.js +19 -0
- package/dist/mvp/server.js +118 -10
- package/dist/retrieval/orchestrator.js +69 -16
- package/dist/retrieval/rankingTrace.js +184 -0
- package/dist/sharing/cli.js +181 -0
- package/dist/sharing/examples.js +136 -0
- package/dist/sharing/fields.js +225 -0
- package/dist/sharing/loop.js +80 -0
- package/dist/sharing/project.js +327 -0
- package/dist/sharing/send.js +466 -0
- package/dist/sharing/state.js +222 -0
- package/dist/sharing/wording.js +48 -0
- package/package.json +1 -1
- package/dist/BUILD.json +0 -6
package/dist/mvp/exportAll.js
CHANGED
|
@@ -14,6 +14,8 @@ const memoryMap_1 = require("../memorymap/memoryMap");
|
|
|
14
14
|
const parser_1 = require("../indexer/parser");
|
|
15
15
|
const handoff_1 = require("../memory/handoff");
|
|
16
16
|
const replacements_1 = require("../memorymap/replacements");
|
|
17
|
+
const settings_1 = require("../learning/settings");
|
|
18
|
+
const store_1 = require("../learning/store");
|
|
17
19
|
// ---------------------------------------------------------------------------
|
|
18
20
|
// export_memory: everything one workspace holds, in one documented Markdown format ("renor-export/v1").
|
|
19
21
|
//
|
|
@@ -41,6 +43,7 @@ exports.NOT_EXPORTED = [
|
|
|
41
43
|
{ kind: 'agentState', what: 'connected agent names, read cursors and live leases (known_agents, agent_cursors, agent_leases)', reason: 'connection state, not memory' },
|
|
42
44
|
{ kind: 'searchHistory', what: 'the promoted history directory that search_history reads (MVP_HISTORY_DIR)', reason: 'configured outside the memory folder' },
|
|
43
45
|
{ kind: 'searchIndex', what: 'the search index and embeddings', reason: 'derived: index_memory rebuilds them from the exported files' },
|
|
46
|
+
{ kind: 'sharingStore', what: 'the sharing choice, sender state and optional text examples in .memory-index/sharing/', reason: 'consent state and sender credentials are local controls, not portable memory; the learningLog manifest field separately exports learning.sqlite' },
|
|
44
47
|
{ kind: 'otherFiles', what: 'files in the memory folder that are not Markdown (.md), dot-folders other than the retired area, and symbolic links', reason: 'the export carries Markdown notes only' },
|
|
45
48
|
];
|
|
46
49
|
/**
|
|
@@ -267,6 +270,19 @@ async function buildSections(ctx, wanted) {
|
|
|
267
270
|
}
|
|
268
271
|
return built;
|
|
269
272
|
}
|
|
273
|
+
/** The portable contents of learning.sqlite, when that optional log exists. */
|
|
274
|
+
function exportLearningLog(memoryDir) {
|
|
275
|
+
const file = (0, settings_1.learningDbPath)(memoryDir);
|
|
276
|
+
const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(), rawQueries: false, create: false });
|
|
277
|
+
if (log === null)
|
|
278
|
+
return null;
|
|
279
|
+
try {
|
|
280
|
+
return { file: '.memory-index/learning.sqlite', data: log.exportLog() };
|
|
281
|
+
}
|
|
282
|
+
finally {
|
|
283
|
+
log.close();
|
|
284
|
+
}
|
|
285
|
+
}
|
|
270
286
|
function sectionHeader(name, count, from) {
|
|
271
287
|
return `\n# ${exports.EXPORT_FORMAT_VERSION} · ${name} (${count} item${count === 1 ? '' : 's'}${from > 0 ? `, continuing from item ${from}` : ''})\n\n`;
|
|
272
288
|
}
|
|
@@ -297,6 +313,9 @@ async function exportEverything(ctx, opts) {
|
|
|
297
313
|
counts,
|
|
298
314
|
sections,
|
|
299
315
|
notExported: notExported(),
|
|
316
|
+
// learning.sqlite is user-owned workspace data, not a rebuildable search index. Export its
|
|
317
|
+
// portable rows (never its private HMAC secret) alongside the Markdown memory document.
|
|
318
|
+
learningLog: exportLearningLog(ctx.memoryDir),
|
|
300
319
|
};
|
|
301
320
|
let document = '';
|
|
302
321
|
const included = [];
|
package/dist/mvp/server.js
CHANGED
|
@@ -57,6 +57,12 @@ const questionSpan_1 = require("../retrieval/questionSpan");
|
|
|
57
57
|
const readerEvents_1 = require("../retrieval/readerEvents");
|
|
58
58
|
const orchestrator_1 = require("../retrieval/orchestrator");
|
|
59
59
|
const buildIdentity_1 = require("../buildIdentity");
|
|
60
|
+
const tap_1 = require("../learning/tap");
|
|
61
|
+
const rankingTrace_1 = require("../retrieval/rankingTrace");
|
|
62
|
+
const examples_1 = require("../sharing/examples");
|
|
63
|
+
const fields_1 = require("../sharing/fields");
|
|
64
|
+
const send_1 = require("../sharing/send");
|
|
65
|
+
const state_1 = require("../sharing/state");
|
|
60
66
|
const hostHeaderValidation_js_1 = require("@modelcontextprotocol/sdk/server/middleware/hostHeaderValidation.js");
|
|
61
67
|
const apiAuth_1 = require("../api/apiAuth");
|
|
62
68
|
const memoryDir_1 = require("./memoryDir");
|
|
@@ -64,6 +70,7 @@ const hosted_1 = require("../importing/hosted");
|
|
|
64
70
|
const exportAll_1 = require("./exportAll");
|
|
65
71
|
const memoryMap_1 = require("../memorymap/memoryMap");
|
|
66
72
|
const idle_1 = require("./idle");
|
|
73
|
+
const dreamer_1 = require("../memory/dreamer/dreamer");
|
|
67
74
|
const activity_1 = require("./activity");
|
|
68
75
|
const storedSize_1 = require("./storedSize");
|
|
69
76
|
const sanitize_1 = require("./sanitize");
|
|
@@ -456,11 +463,12 @@ function withAnnotationTitle(config) {
|
|
|
456
463
|
return { ...config, annotations: { ...config.annotations, title: config.title } };
|
|
457
464
|
}
|
|
458
465
|
/**
|
|
459
|
-
* The
|
|
460
|
-
*
|
|
461
|
-
* differ, registeredToolEntry throws when an
|
|
466
|
+
* The tools only the dashboard may list or run: one erases retired memories for good, the other reads and sets the workspace's
|
|
467
|
+
* choice to share learning data. Each registerTool call below spells its name as a literal, because the docs binding and the
|
|
468
|
+
* hosted-tool tests read the registry off the literal call sites; if the two ever differ, registeredToolEntry throws when an
|
|
469
|
+
* agent's server is built.
|
|
462
470
|
*/
|
|
463
|
-
const
|
|
471
|
+
const OWNER_ONLY_TOOLS = ['delete_retired_memory', 'sharing_settings'];
|
|
464
472
|
/** The tools agents use to reach each other. A directory client is offered none of them; registeredToolEntry throws if one is renamed. */
|
|
465
473
|
const HIVE_TOOLS = ['whats_new', 'ask_agent', 'list_requests', 'answer_request', 'close_request'];
|
|
466
474
|
/**
|
|
@@ -478,7 +486,7 @@ function registeredToolEntry(server, name) {
|
|
|
478
486
|
* binds one transport per Protocol instance, so SSE sessions each get
|
|
479
487
|
* their own server (bounded, in-process); JSON mode shares one.
|
|
480
488
|
*/
|
|
481
|
-
function buildMcpServer(db, config, engine, idle, digests, usefulness, connection, announceFirst, telemetrySession = null, dashboard = false, directory = false) {
|
|
489
|
+
function buildMcpServer(db, config, engine, idle, digests, usefulness, connection, announceFirst, telemetrySession = null, dashboard = false, directory = false, dreamer = null) {
|
|
482
490
|
// RENOR_TOOL_OUTPUT_SCHEMAS, read once for this server: its tools list an outputSchema exactly when its answers carry
|
|
483
491
|
// structuredContent.
|
|
484
492
|
const withSchemas = toolOutputSchemasEnabled();
|
|
@@ -531,6 +539,16 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
|
|
|
531
539
|
function toolResultOk(result) {
|
|
532
540
|
return !(result && result.isError);
|
|
533
541
|
}
|
|
542
|
+
// The learning log (RENOR_LEARNING_LOG, src/learning): null unless it is on, and then one record per successful call of the
|
|
543
|
+
// tools it covers. The observer reads the reply before it is decorated and never changes it.
|
|
544
|
+
const learner = (0, tap_1.learningObserver)(config.learning, {
|
|
545
|
+
actor: dashboard ? 'dashboard' : 'agent',
|
|
546
|
+
session: telemetrySession,
|
|
547
|
+
model: () => engine.modelKey ?? null,
|
|
548
|
+
agent: extra => agentOf(extra, config),
|
|
549
|
+
// The owner's choice to share, read at each call: recording turns on when they choose a level and off when they choose off.
|
|
550
|
+
sharing: config.sharingDir === undefined ? undefined : (0, examples_1.sharingHook)(config.sharingDir),
|
|
551
|
+
});
|
|
534
552
|
/** Privacy-safe: tool name / ok / ms and, on a refusal, its code from the closed set — never args or memory content. */
|
|
535
553
|
function tracedTool(tool, handler) {
|
|
536
554
|
// On-demand: a running tool call (a write, an index job) holds the engine awake
|
|
@@ -547,8 +565,12 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
|
|
|
547
565
|
return report && Object.keys(report).length > 0 ? { phases: report } : {};
|
|
548
566
|
};
|
|
549
567
|
try {
|
|
550
|
-
|
|
568
|
+
// The ranker's terms are collected only for the learning log (rankingTrace.ts): with the log off no scope is opened.
|
|
569
|
+
const trace = learner ? new rankingTrace_1.RankingTrace(config.memoryDir) : undefined;
|
|
570
|
+
const invoke = () => (phases ? phases.run(() => run(...args)) : run(...args));
|
|
571
|
+
const result = await (trace ? (0, rankingTrace_1.withRankingTrace)(trace, invoke) : invoke());
|
|
551
572
|
const ok = toolResultOk(result);
|
|
573
|
+
learner?.observe(tool, args[0], result, ok, Date.now() - started, args[1], trace);
|
|
552
574
|
(0, telemetry_1.emitTelemetry)({
|
|
553
575
|
event: 'tool_use',
|
|
554
576
|
tool,
|
|
@@ -687,6 +709,7 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
|
|
|
687
709
|
// FULL_VAULT_CHECK_MS (see engineIndexChanged).
|
|
688
710
|
let refresh = {};
|
|
689
711
|
if (receipt.action !== 'nochange') {
|
|
712
|
+
dreamer?.noteFactSave();
|
|
690
713
|
try {
|
|
691
714
|
const pass = await (0, engine_1.engineIndexChanged)(db, config, engine, undefined, {
|
|
692
715
|
interactive: true,
|
|
@@ -854,6 +877,8 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
|
|
|
854
877
|
duplicates = (0, factConflicts_1.possibleDuplicates)(db, receipt, receipt.created?.factId ?? null, (0, lifecycle_1.scopeFilterFor)(args.scope), readerOf(extra));
|
|
855
878
|
}
|
|
856
879
|
catch { /* no hint */ }
|
|
880
|
+
if (receipt.action !== 'nochange')
|
|
881
|
+
dreamer?.noteFactSave();
|
|
857
882
|
return jsonResult({
|
|
858
883
|
ok: receipt.ok,
|
|
859
884
|
action: receipt.action,
|
|
@@ -981,6 +1006,7 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
|
|
|
981
1006
|
counts: zod_1.z.record(zod_1.z.string(), zod_1.z.unknown()),
|
|
982
1007
|
sections: zod_1.z.array(zod_1.z.object({ name: zod_1.z.string(), items: zod_1.z.number().nullable(), events: zod_1.z.number().nullable().optional() })),
|
|
983
1008
|
notExported: zod_1.z.array(zod_1.z.object({ kind: zod_1.z.string(), what: zod_1.z.string(), reason: zod_1.z.string() })),
|
|
1009
|
+
learningLog: zod_1.z.union([zod_1.z.null(), zod_1.z.object({ file: zod_1.z.string(), data: zod_1.z.record(zod_1.z.string(), zod_1.z.unknown()) })]),
|
|
984
1010
|
included: zod_1.z.array(zod_1.z.string()),
|
|
985
1011
|
truncated: zod_1.z.boolean(),
|
|
986
1012
|
nextSection: zod_1.z.string().nullable(),
|
|
@@ -1573,7 +1599,76 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
|
|
|
1573
1599
|
}
|
|
1574
1600
|
});
|
|
1575
1601
|
if (!dashboard)
|
|
1576
|
-
registeredToolEntry(server,
|
|
1602
|
+
registeredToolEntry(server, 'delete_retired_memory').enabled = false;
|
|
1603
|
+
// Only for the owner, from the dashboard: the workspace's choice to share learning data with Renor (src/sharing). The consent
|
|
1604
|
+
// record the owner's click writes in the control plane (set_learning_consent) is the proof; THIS is what the tenant obeys, so the
|
|
1605
|
+
// dashboard calls it right after the record is written, and recording and sending start or stop on the next call. It guards
|
|
1606
|
+
// itself exactly as delete_retired_memory does, and for an agent it is neither listed nor callable.
|
|
1607
|
+
server.registerTool('sharing_settings', {
|
|
1608
|
+
title: 'Learning-data sharing choice (owner only)',
|
|
1609
|
+
annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true },
|
|
1610
|
+
description: 'Reads, sets or deletes this workspace\'s choice to share learning data with Renor. action "get" returns the choice; "set" needs level (off, signals or data) and, for signals or data, adult: true and the notice_version the owner saw; "delete" asks Renor to delete what it holds for this workspace and removes the local sharing files. Off is the default.',
|
|
1611
|
+
inputSchema: {
|
|
1612
|
+
action: zod_1.z.enum(['get', 'set', 'delete']).describe('get, set or delete'),
|
|
1613
|
+
level: zod_1.z.enum(fields_1.SHARING_LEVELS).optional().describe('With set: off, signals or data'),
|
|
1614
|
+
notice_version: zod_1.z.string().min(1).max(80).optional().describe('With set: the id of the words the owner saw'),
|
|
1615
|
+
adult: zod_1.z.boolean().optional().describe('With set to signals or data: the owner confirmed they are 18 or older'),
|
|
1616
|
+
},
|
|
1617
|
+
outputSchema: zod_1.z.looseObject({
|
|
1618
|
+
ok: zod_1.z.boolean(),
|
|
1619
|
+
action: zod_1.z.enum(['get', 'set', 'delete']),
|
|
1620
|
+
level: zod_1.z.enum(fields_1.SHARING_LEVELS),
|
|
1621
|
+
effective: zod_1.z.enum(fields_1.SHARING_LEVELS),
|
|
1622
|
+
changedAt: zod_1.z.string().nullable(),
|
|
1623
|
+
noticeVersion: zod_1.z.string().nullable(),
|
|
1624
|
+
remoteDeleted: zod_1.z.boolean().nullable(),
|
|
1625
|
+
contributorId: zod_1.z.string().nullable(),
|
|
1626
|
+
}),
|
|
1627
|
+
}, async (args, extra) => {
|
|
1628
|
+
try {
|
|
1629
|
+
if (!isDashboardCall(extra?.requestInfo?.headers, config)) {
|
|
1630
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'sharing_settings is only available to the owner, from the Renor dashboard');
|
|
1631
|
+
}
|
|
1632
|
+
const dir = config.sharingDir;
|
|
1633
|
+
if (dir === undefined)
|
|
1634
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'sharing is not configured on this server');
|
|
1635
|
+
let remoteDeleted = null;
|
|
1636
|
+
if (args.action === 'set') {
|
|
1637
|
+
if (args.level === undefined)
|
|
1638
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'set needs level');
|
|
1639
|
+
if (args.level !== 'off' && (args.adult !== true || args.notice_version === undefined)) {
|
|
1640
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'sharing signals or data needs adult: true and the notice_version the owner saw');
|
|
1641
|
+
}
|
|
1642
|
+
const result = await (0, send_1.chooseLevel)({
|
|
1643
|
+
dir, level: args.level, wording: args.notice_version ?? 'dashboard', adult: args.adult === true, source: 'dashboard', hosted: true,
|
|
1644
|
+
});
|
|
1645
|
+
remoteDeleted = result.remoteDeleted;
|
|
1646
|
+
}
|
|
1647
|
+
else if (args.action === 'delete') {
|
|
1648
|
+
const result = await (0, send_1.deleteAllShared)({ dir, hosted: true });
|
|
1649
|
+
remoteDeleted = result.remote.requested ? result.remote.ok : null;
|
|
1650
|
+
}
|
|
1651
|
+
const record = (0, state_1.readConsent)(dir);
|
|
1652
|
+
// The name the control plane ties to this workspace, so the endpoint can check the control plane's consent record before it
|
|
1653
|
+
// stores a row. It is the hash of the sender's secret; made on first use only for a workspace that has chosen to share.
|
|
1654
|
+
const sharing = record !== null && record.level !== 'off' && record.adult;
|
|
1655
|
+
return jsonResult({
|
|
1656
|
+
ok: true,
|
|
1657
|
+
action: args.action,
|
|
1658
|
+
contributorId: sharing ? (0, state_1.contributorId)((0, state_1.ensureState)(dir).secret) : null,
|
|
1659
|
+
level: record?.level ?? 'off',
|
|
1660
|
+
effective: (0, state_1.effectiveLevel)(dir),
|
|
1661
|
+
changedAt: record?.changedAt ?? null,
|
|
1662
|
+
noticeVersion: record?.wording ?? null,
|
|
1663
|
+
remoteDeleted,
|
|
1664
|
+
});
|
|
1665
|
+
}
|
|
1666
|
+
catch (err) {
|
|
1667
|
+
return toolError(err);
|
|
1668
|
+
}
|
|
1669
|
+
});
|
|
1670
|
+
if (!dashboard)
|
|
1671
|
+
registeredToolEntry(server, 'sharing_settings').enabled = false;
|
|
1577
1672
|
server.registerTool('whats_new', {
|
|
1578
1673
|
title: "What's new for me",
|
|
1579
1674
|
// It moves this connection's mark, so it is a writing tool, not read-only: a second call returns less.
|
|
@@ -1810,7 +1905,8 @@ function buildMcpServer(db, config, engine, idle, digests, usefulness, connectio
|
|
|
1810
1905
|
const forDirectory = directory || isDirectoryCall(headers, config);
|
|
1811
1906
|
const hidden = new Set();
|
|
1812
1907
|
if (!isDashboardCall(headers, config))
|
|
1813
|
-
|
|
1908
|
+
for (const name of OWNER_ONLY_TOOLS)
|
|
1909
|
+
hidden.add(name);
|
|
1814
1910
|
if (forDirectory)
|
|
1815
1911
|
for (const name of [...HIVE_TOOLS, ...directorySurface_1.DIRECTORY_HIDDEN_TOOLS])
|
|
1816
1912
|
hidden.add(name);
|
|
@@ -1895,7 +1991,15 @@ function createMvpApp(config, engineOverride) {
|
|
|
1895
1991
|
digests?.worker.start();
|
|
1896
1992
|
const usefulness = new readUsefulness_1.ReadUsefulness();
|
|
1897
1993
|
const connection = (0, connectionStatus_1.connectionStatusFromEnv)(process.env);
|
|
1898
|
-
|
|
1994
|
+
// The consolidation pass (RENOR_CONSOLIDATE, default off): null, and nothing else changes, unless it is switched on.
|
|
1995
|
+
const dreamer = (0, dreamer_1.createDreamer)(process.env, {
|
|
1996
|
+
db,
|
|
1997
|
+
vaultPath: config.memoryDir,
|
|
1998
|
+
ledgerRelPath: config.ledgerRelPath,
|
|
1999
|
+
index: () => (0, engine_1.engineIndexChanged)(db, config, engine),
|
|
2000
|
+
hold: run => { activity.track(() => run).catch(() => undefined); },
|
|
2001
|
+
});
|
|
2002
|
+
const server = buildMcpServer(db, config, engine, activity, digests, usefulness, connection, false, null, false, false, dreamer);
|
|
1899
2003
|
// An index an earlier build wrote holds the engine's own daily-note headers as facts, and a note that has not changed
|
|
1900
2004
|
// is never rebuilt, so they are removed here, before anything is read. A failure must not keep a workspace from serving.
|
|
1901
2005
|
try {
|
|
@@ -2312,7 +2416,7 @@ function createMvpApp(config, engineOverride) {
|
|
|
2312
2416
|
// calls, isPlatformInitialize) gets no id: it still counts as a connection, but it writes no first call and its mcp_init
|
|
2313
2417
|
// carries no id for the recall share to join, so the share counts agents only.
|
|
2314
2418
|
const telemetrySession = (0, telemetry_1.isPlatformInitialize)(req.body) ? null : crypto_1.default.randomBytes(8).toString('hex');
|
|
2315
|
-
const sessionServer = buildMcpServer(db, config, engine, activity, digests, usefulness, connection, true, telemetrySession, isDashboardCall(req.headers, config), isDirectoryCall(req.headers, config));
|
|
2419
|
+
const sessionServer = buildMcpServer(db, config, engine, activity, digests, usefulness, connection, true, telemetrySession, isDashboardCall(req.headers, config), isDirectoryCall(req.headers, config), dreamer);
|
|
2316
2420
|
transport.onclose = () => {
|
|
2317
2421
|
sseSessions.delete(sessionId);
|
|
2318
2422
|
(0, telemetry_1.emitTelemetry)({ event: 'session', kind: 'mcp_close' });
|
|
@@ -2404,9 +2508,12 @@ function createMvpApp(config, engineOverride) {
|
|
|
2404
2508
|
idle,
|
|
2405
2509
|
digests,
|
|
2406
2510
|
reindexPending: pendingPass,
|
|
2511
|
+
dreamer,
|
|
2407
2512
|
close: () => {
|
|
2408
2513
|
closing ??= (async () => {
|
|
2409
2514
|
// First: it is background work, and what it has started it finishes before the database closes.
|
|
2515
|
+
dreamer?.stop();
|
|
2516
|
+
await dreamer?.inFlight()?.catch(() => undefined);
|
|
2410
2517
|
(0, engine_1.runningEmbeddingRebuild)(engine)?.abort();
|
|
2411
2518
|
(0, engine_1.runningBackgroundIndex)(db)?.abort();
|
|
2412
2519
|
// The digest worker uses the database too: its sweep in flight ends before the database closes.
|
|
@@ -2557,6 +2664,7 @@ async function startMvpService(config, engineOverride) {
|
|
|
2557
2664
|
// Restart recovery: replay the authoritative ledger before readiness.
|
|
2558
2665
|
// Start-up is the one place a torn final event is set aside: nothing else is appending to this ledger yet.
|
|
2559
2666
|
await (0, lifecycle_1.replayLedger)(mvp.db, config.memoryDir, config.ledgerRelPath, { setTornTailAside: true });
|
|
2667
|
+
mvp.dreamer?.start();
|
|
2560
2668
|
// The notes of a folder that moved were removed from the index with their old paths: they are indexed at their
|
|
2561
2669
|
// current paths before the service listens, so the first answer has them. The mark is cleared only by a pass with no
|
|
2562
2670
|
// failed file. A file whose embedding the provider still refuses after its retries counts as failed: it keeps its
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ENTITY_LANE_RESERVED = exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
|
|
3
|
+
exports.ENTITY_LANE_RESERVED = exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
|
|
4
4
|
exports.searchBestFirstEnabled = searchBestFirstEnabled;
|
|
5
5
|
exports.parseQueryDateRef = parseQueryDateRef;
|
|
6
6
|
exports.queryDateRefMs = queryDateRefMs;
|
|
@@ -41,6 +41,7 @@ const keyword_1 = require("./keyword");
|
|
|
41
41
|
const queryHelpers_1 = require("./queryHelpers");
|
|
42
42
|
const ledgerPresentation_2 = require("./ledgerPresentation");
|
|
43
43
|
const plainWording_1 = require("./plainWording");
|
|
44
|
+
const rankingTrace_1 = require("./rankingTrace");
|
|
44
45
|
function envFlagOn(name) {
|
|
45
46
|
const v = (process.env[name] ?? '').trim().toLowerCase();
|
|
46
47
|
return v === '1' || v === 'true' || v === 'yes' || v === 'on';
|
|
@@ -610,20 +611,31 @@ function resolveRankedExpansion(env = process.env) {
|
|
|
610
611
|
};
|
|
611
612
|
}
|
|
612
613
|
/**
|
|
613
|
-
*
|
|
614
|
-
*
|
|
615
|
-
|
|
616
|
-
|
|
614
|
+
* The longest conversation statement the RANKED expansion adds when nothing says otherwise: 400 tokens. The former expansion
|
|
615
|
+
* (a command line that does not declare a calibrated model) keeps RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120).
|
|
616
|
+
*/
|
|
617
|
+
exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = 400;
|
|
618
|
+
/**
|
|
619
|
+
* RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS (default 400 since the flip of 2026-10-25): the longest conversation statement,
|
|
620
|
+
* in tokens, that the RANKED expansion alone may add. Unset or empty: 400, unless RENOR_SESSION_EXPAND_STATEMENT_TOKENS is set,
|
|
621
|
+
* which an operator's explicit value keeps winning as it always did. A number of 1 or more: that limit, over the general one.
|
|
622
|
+
* The off switch: `0`, a negative number or anything that is not a number returns null, and the ranked expansion then takes
|
|
623
|
+
* RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120 unset), as it did before the flip. The former expansion and the documents are
|
|
624
|
+
* never touched by it.
|
|
617
625
|
*
|
|
618
|
-
* Why
|
|
619
|
-
*
|
|
620
|
-
*
|
|
621
|
-
*
|
|
622
|
-
*
|
|
623
|
-
*
|
|
626
|
+
* Why 400: the session expansion skips a conversation statement over its limit, whole, and the long user turn that answers a
|
|
627
|
+
* question is often just over 120 tokens (501 to 700 characters: missing from the page 5 times in 21, against 6 in 777 for
|
|
628
|
+
* shorter ones, on the Haiku default run of 2026-10-23). The chunker leaves no section over 400 tokens as one chunk, so 400 is
|
|
629
|
+
* the largest value that changes anything. On the official LongMemEval-S instrument with a Haiku 5.5 reader the ranked
|
|
630
|
+
* expansion with 400 scored 433 of 500 and with 120 (the control) 431, against 418, 417 and 423 for the command-line default;
|
|
631
|
+
* the four pages only the limit completes are right 4 of 4 with it and 0 of 4 without, and the page is 5.5% smaller:
|
|
632
|
+
* engine/docs/evals/results/ranked-statement-limit-20261024/ and LEDGER sections BE and BF.
|
|
624
633
|
*/
|
|
625
634
|
function rankedStatementTokens(env = process.env) {
|
|
626
|
-
const
|
|
635
|
+
const raw = (env.RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS ?? '').trim();
|
|
636
|
+
if (raw === '')
|
|
637
|
+
return (env.RENOR_SESSION_EXPAND_STATEMENT_TOKENS ?? '').trim() === '' ? exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS : null;
|
|
638
|
+
const value = Number(raw);
|
|
627
639
|
return Number.isFinite(value) && value >= 1 ? Math.floor(value) : null;
|
|
628
640
|
}
|
|
629
641
|
/**
|
|
@@ -856,6 +868,8 @@ function reserveLanePlaces(ranked, topK, laneTerms) {
|
|
|
856
868
|
/** The window of hits of a read: scored, with the graph of its notes, their facts and dates, live and capped per note. Waits for nothing. */
|
|
857
869
|
function rankWindow(read) {
|
|
858
870
|
const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap } = read;
|
|
871
|
+
// Set only while the learning log collects the ranker's terms (rankingTrace.ts); otherwise every use below is skipped.
|
|
872
|
+
const trace = (0, rankingTrace_1.currentRankingTrace)();
|
|
859
873
|
// Raw cosines, kept only for a caller that asked to see them (get_context's relevance signal).
|
|
860
874
|
const cosineByChunk = opts.probe ? new Map(semanticHits.map(h => [h.chunkId, h.score])) : null;
|
|
861
875
|
const entityHit = (0, entityLookup_1.lookupEntity)(db, query.trim());
|
|
@@ -877,8 +891,11 @@ function rankWindow(read) {
|
|
|
877
891
|
const scored = semanticHits.map(h => {
|
|
878
892
|
let score = h.score;
|
|
879
893
|
let reason = 'semantic';
|
|
894
|
+
const terms = trace ? (0, rankingTrace_1.newRankingTerms)() : null;
|
|
880
895
|
if (isSpeakerStub(h.text)) {
|
|
881
896
|
score -= 0.5;
|
|
897
|
+
if (terms)
|
|
898
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('stub')] -= 0.5;
|
|
882
899
|
reason = 'semantic+stub';
|
|
883
900
|
}
|
|
884
901
|
else {
|
|
@@ -886,21 +903,30 @@ function rankWindow(read) {
|
|
|
886
903
|
if (boost > 0) {
|
|
887
904
|
score += boost;
|
|
888
905
|
reason += '+content';
|
|
906
|
+
if (terms)
|
|
907
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('content')] += boost;
|
|
889
908
|
}
|
|
890
909
|
}
|
|
891
910
|
if (priorityPaths.some(p => h.notePath.includes('/' + p) || h.notePath.startsWith(p))) {
|
|
892
911
|
score += 0.1;
|
|
893
912
|
reason += '+priority';
|
|
913
|
+
if (terms)
|
|
914
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('priority')] += 0.1;
|
|
894
915
|
}
|
|
895
916
|
if (entityHit?.sourceNote === h.notePath) {
|
|
896
917
|
score += 0.2;
|
|
897
918
|
reason += '+entity';
|
|
919
|
+
if (terms)
|
|
920
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('entity')] += 0.2;
|
|
898
921
|
}
|
|
899
922
|
if (laneTerms.length > 0) {
|
|
900
923
|
const share = (0, plainWording_1.laneTermShare)(h.text, laneTerms);
|
|
901
924
|
if (share > 0) {
|
|
902
|
-
|
|
925
|
+
const boost = plainWording_1.ENTITY_LANE_BOOST * share;
|
|
926
|
+
score += boost;
|
|
903
927
|
reason += '+entity-lane';
|
|
928
|
+
if (terms)
|
|
929
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('entity_lane')] += boost;
|
|
904
930
|
}
|
|
905
931
|
}
|
|
906
932
|
// Time-decay recency: +0.1 for today, linearly decaying to 0 over 30 days.
|
|
@@ -914,9 +940,12 @@ function rankWindow(read) {
|
|
|
914
940
|
if (recencyBoost > 0.001) {
|
|
915
941
|
score += recencyBoost;
|
|
916
942
|
reason += '+recency';
|
|
943
|
+
if (terms)
|
|
944
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('recency')] += recencyBoost;
|
|
917
945
|
}
|
|
918
946
|
}
|
|
919
|
-
|
|
947
|
+
const out = { ...h, score, reason, session_date };
|
|
948
|
+
return terms ? (0, rankingTrace_1.attachRankingTerms)(out, terms) : out;
|
|
920
949
|
});
|
|
921
950
|
// Graph expansion
|
|
922
951
|
const hitNotePaths = [...new Set(semanticHits.map(h => h.notePath))];
|
|
@@ -1017,6 +1046,8 @@ function rankWindow(read) {
|
|
|
1017
1046
|
(f.predicate ?? '').toLowerCase().includes(w)));
|
|
1018
1047
|
if (factMatch) {
|
|
1019
1048
|
hit.score += 0.15;
|
|
1049
|
+
if (trace)
|
|
1050
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'fact', 0.15);
|
|
1020
1051
|
hit.reason += '+fact';
|
|
1021
1052
|
}
|
|
1022
1053
|
// As-of lane: the query names a past date, so hits carrying the same
|
|
@@ -1027,11 +1058,15 @@ function rankWindow(read) {
|
|
|
1027
1058
|
const lowerText = hit.text.toLowerCase();
|
|
1028
1059
|
if (lowerText.includes(yearToken)) {
|
|
1029
1060
|
hit.score += 0.08;
|
|
1061
|
+
if (trace)
|
|
1062
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'asof_year', 0.08);
|
|
1030
1063
|
hit.reason += '+asof-year';
|
|
1031
1064
|
if (asofRef.month !== undefined) {
|
|
1032
1065
|
const monthName = Object.keys(ASOF_MONTHS).find(m => ASOF_MONTHS[m] === asofRef.month);
|
|
1033
1066
|
if (monthName && lowerText.includes(monthName)) {
|
|
1034
1067
|
hit.score += 0.04;
|
|
1068
|
+
if (trace)
|
|
1069
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'asof_month', 0.04);
|
|
1035
1070
|
hit.reason += '+asof-month';
|
|
1036
1071
|
}
|
|
1037
1072
|
}
|
|
@@ -1054,6 +1089,8 @@ function rankWindow(read) {
|
|
|
1054
1089
|
// Must clear recency (+0.1 max) + fact boost (+0.15) the current note
|
|
1055
1090
|
// typically carries, or past-intent queries can never win.
|
|
1056
1091
|
hit.score += 0.3;
|
|
1092
|
+
if (trace)
|
|
1093
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'historical_intent', 0.3);
|
|
1057
1094
|
hit.reason += '+historical-intent';
|
|
1058
1095
|
}
|
|
1059
1096
|
}
|
|
@@ -1066,10 +1103,14 @@ function rankWindow(read) {
|
|
|
1066
1103
|
if (stale.length >= 3 && text.includes(stale)) {
|
|
1067
1104
|
if (pastIntent) {
|
|
1068
1105
|
hit.score += 0.1;
|
|
1106
|
+
if (trace)
|
|
1107
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'past_fact', 0.1);
|
|
1069
1108
|
hit.reason += '+historical-intent';
|
|
1070
1109
|
}
|
|
1071
1110
|
else {
|
|
1072
1111
|
hit.score -= 0.15;
|
|
1112
|
+
if (trace)
|
|
1113
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'stale_fact', -0.15);
|
|
1073
1114
|
hit.reason += '+stale-fact';
|
|
1074
1115
|
}
|
|
1075
1116
|
break; // one demotion per hit; stale signal already applied
|
|
@@ -1086,6 +1127,8 @@ function rankWindow(read) {
|
|
|
1086
1127
|
const cur = (f.object_text ?? '').trim().toLowerCase();
|
|
1087
1128
|
if (cur.length >= 3 && text.includes(cur)) {
|
|
1088
1129
|
hit.score += 0.1;
|
|
1130
|
+
if (trace)
|
|
1131
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'current_fact', 0.1);
|
|
1089
1132
|
hit.reason += '+current-fact';
|
|
1090
1133
|
break;
|
|
1091
1134
|
}
|
|
@@ -1094,6 +1137,8 @@ function rankWindow(read) {
|
|
|
1094
1137
|
}
|
|
1095
1138
|
}
|
|
1096
1139
|
const ranked = dedupeHits(scored.sort(semantic_1.byScoreThenPlace));
|
|
1140
|
+
if (trace)
|
|
1141
|
+
ranked.forEach((hit, index) => trace.pool(hit, index + 1));
|
|
1097
1142
|
// Fail-open floor. The stub filter must never empty the window: a zero-hit
|
|
1098
1143
|
// retrieval is a hard abort on the product path, because the benchmark client
|
|
1099
1144
|
// refuses to score a case with no hits ("Retrieval precondition failed: public
|
|
@@ -1112,7 +1157,12 @@ function rankWindow(read) {
|
|
|
1112
1157
|
const mixed = (0, ledgerPresentation_1.dropLedgerPreamble)((0, ledgerLiveness_1.resolveLedgerHits)([...datedFactHits, ...contentful.filter(h => !seen.has(h.text))], (0, ledgerLiveness_1.closedLedgerEvents)(db), opts.includeHistory === true), opts.isLedgerNote);
|
|
1113
1158
|
const diversityCap = resolveSessionDiversityCap(process.env, opts.noteCapWhenUnset);
|
|
1114
1159
|
// RENOR_NEAR_DUP_CAP (default off): near-copies are capped before the per-note cap, on the visible text.
|
|
1115
|
-
|
|
1160
|
+
const windowed = reserveLanePlaces(applySessionDiversity((0, plainWording_1.applyNearDupCap)(mixed, nearDupCap, contentfulText), diversityCap, topK), topK, laneTerms);
|
|
1161
|
+
// The window's candidates are recorded before they are copied into the plain hits below, which carry no terms.
|
|
1162
|
+
if (trace)
|
|
1163
|
+
for (const hit of windowed)
|
|
1164
|
+
trace.touch(hit);
|
|
1165
|
+
let hits = windowed.map(h => ({
|
|
1116
1166
|
notePath: h.notePath,
|
|
1117
1167
|
chunkId: h.chunkId,
|
|
1118
1168
|
headingPath: h.headingPath,
|
|
@@ -1167,5 +1217,8 @@ async function finishContext(read) {
|
|
|
1167
1217
|
const newestFirst = !bestFirst && newestFirstEnabled() && !(temporal && pastIntent) && !asofPast;
|
|
1168
1218
|
const rankedBy = newestFirst ? 'session_date' : 'score';
|
|
1169
1219
|
contentfulHeld.clear(); // nothing outlives the read it was worked out for
|
|
1170
|
-
|
|
1220
|
+
// The order the reply presents its hits in, so a candidate's place in it is recorded for the learning log (rankingTrace.ts).
|
|
1221
|
+
const presented = orderHits(hits, rankedBy);
|
|
1222
|
+
(0, rankingTrace_1.currentRankingTrace)()?.presented(presented);
|
|
1223
|
+
return { query, hits: presented, rankedBy };
|
|
1171
1224
|
}
|