renor-engine 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -661
- package/NOTICE +8 -6
- package/README.md +13 -8
- package/dist/.tsbuildinfo +1 -0
- package/dist/api/server.js +10 -8
- package/dist/buildIdentity.js +18 -1
- package/dist/callPhases.js +82 -0
- package/dist/cli.js +36 -1
- package/dist/context/rowStore.js +1 -1
- package/dist/conversation/parser.js +40 -36
- package/dist/db/client.js +6 -4
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +24 -4
- package/dist/db/migrations.js +30 -2
- package/dist/db/sqlite.js +396 -0
- package/dist/digests/digestCache.js +6 -2
- package/dist/digests/digests.js +3 -3
- package/dist/digests/worker.js +3 -1
- package/dist/embeddings/caching.js +5 -5
- package/dist/embeddings/local.js +5 -3
- package/dist/embeddings/openai.js +4 -3
- package/dist/embeddings/simdScan.js +3 -1
- package/dist/embeddings/vectorIndex.js +0 -1
- package/dist/embeddings/vertex.js +123 -13
- package/dist/eval/cacheReport.js +7 -0
- package/dist/eval/context700-candidate.js +5 -5
- package/dist/eval/evalCache.js +46 -3
- package/dist/eval/indexSnapshots.js +549 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/runManifest.js +6 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +0 -3
- package/dist/evidence/evidenceCache.js +3 -3
- package/dist/evidence/ollamaChat.js +1 -1
- package/dist/evidence/options.js +46 -25
- package/dist/evidence/secondPass.js +8 -170
- package/dist/extraction/factExtractor.js +1 -1
- package/dist/extraction/jevDecisionModel.js +307 -0
- package/dist/extraction/preferenceExtractor.js +1 -4
- package/dist/extraction/secretGuard.js +7 -3
- package/dist/extraction/typedFacts.js +1 -1
- package/dist/importing/apply.js +42 -14
- package/dist/importing/cli.js +9 -8
- package/dist/importing/existing.js +2 -0
- package/dist/importing/facts.js +0 -2
- package/dist/importing/plan.js +0 -5
- package/dist/importing/source.js +1 -1
- package/dist/indexer/parser.js +4 -1
- package/dist/indexer/pipeline.js +99 -279
- package/dist/indexer/walker.js +11 -25
- package/dist/learning/cli.js +113 -0
- package/dist/learning/schema.js +144 -0
- package/dist/learning/settings.js +45 -0
- package/dist/learning/store.js +267 -0
- package/dist/learning/tap.js +401 -0
- package/dist/mcp/server.js +36 -6
- package/dist/memory/dreamer/adjudicate.js +215 -0
- package/dist/memory/dreamer/apply.js +120 -0
- package/dist/memory/dreamer/budget.js +64 -0
- package/dist/memory/dreamer/candidates.js +149 -0
- package/dist/memory/dreamer/collection.js +104 -0
- package/dist/memory/dreamer/dreamer.js +352 -0
- package/dist/memory/dreamer/review.js +111 -0
- package/dist/memory/factConflicts.js +2 -0
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/lifecycle.js +338 -80
- package/dist/memory/sourceName.js +77 -1
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memory/writer.js +3 -2
- package/dist/memorymap/memoryMap.js +39 -13
- package/dist/memorymap/replacements.js +8 -3
- package/dist/mvp/agentMemory.js +109 -50
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/cli.js +14 -1
- package/dist/mvp/compileCache.js +3 -10
- package/dist/mvp/compileSeed.js +9 -3
- package/dist/mvp/config.js +4 -0
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +39 -26
- package/dist/mvp/exportAll.js +36 -2
- package/dist/mvp/factMarkers.js +18 -2
- package/dist/mvp/history.js +5 -4
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/server.js +759 -136
- package/dist/mvp/serverCard.js +12 -3
- package/dist/mvp/telemetry.js +35 -2
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/entityLookup.js +53 -4
- package/dist/retrieval/orchestrator.js +148 -10
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/rankingTrace.js +184 -0
- package/dist/retrieval/readerLayout.js +143 -224
- package/dist/router/routerCache.js +2 -2
- package/dist/sharing/cli.js +181 -0
- package/dist/sharing/examples.js +136 -0
- package/dist/sharing/fields.js +225 -0
- package/dist/sharing/loop.js +80 -0
- package/dist/sharing/project.js +327 -0
- package/dist/sharing/send.js +466 -0
- package/dist/sharing/state.js +222 -0
- package/dist/sharing/wording.js +48 -0
- package/dist/watcher/watcher.js +5 -3
- package/package.json +5 -7
- package/dist/BUILD.json +0 -6
- package/dist/importing/index.js +0 -29
|
@@ -33,6 +33,9 @@ exports.hiveIdentity = hiveIdentity;
|
|
|
33
33
|
exports.productFromUserAgent = productFromUserAgent;
|
|
34
34
|
exports.isImportSource = isImportSource;
|
|
35
35
|
exports.savedSource = savedSource;
|
|
36
|
+
exports.reservedCallerSource = reservedCallerSource;
|
|
37
|
+
exports.reservedSourceIn = reservedSourceIn;
|
|
38
|
+
exports.isReservedSource = isReservedSource;
|
|
36
39
|
exports.authorMarker = authorMarker;
|
|
37
40
|
const hiveNameHeader_cjs_1 = require("./hiveNameHeader.cjs");
|
|
38
41
|
/** The longest source a write stores, in characters (code points). */
|
|
@@ -191,10 +194,83 @@ function savedSource(source) {
|
|
|
191
194
|
const name = cleanSource(source);
|
|
192
195
|
return isImportSource(name) ? `import-${name.slice(exports.IMPORT_SOURCE_PREFIX.length)}` : name;
|
|
193
196
|
}
|
|
197
|
+
/**
|
|
198
|
+
* The identities no caller may write as: the roles a model reads as authority (system, developer, assistant), Renor itself,
|
|
199
|
+
* and the workspace's admin, owner and tools. A typed source or a User-Agent product that names one is refused or dropped
|
|
200
|
+
* (mvp/server.ts callerSource and sourceOf), and a trusted connection name that names one is unnamed.
|
|
201
|
+
*/
|
|
202
|
+
const RESERVED_SOURCE_IDENTITIES = new Set(['system', 'developer', 'assistant', 'renor', 'admin', 'owner', 'tool']);
|
|
203
|
+
/**
|
|
204
|
+
* The reserved identities a name a CLIENT chooses may not even start with: `System Message`, `system-prompt`,
|
|
205
|
+
* `developer note`, `renor-engine` and `Renor Support` read as the role or as Renor. Only a whole first word counts
|
|
206
|
+
* (`system2`, `ecosystem` and `my-system` are ordinary names), and admin, owner and tool are reserved as whole names only
|
|
207
|
+
* (`Admin Bot`, `toolbox`). A name the OWNER chose (a gateway key's, from the dashboard) is held to the exact rule only, so a
|
|
208
|
+
* key called `Developer Laptop` or `Renor CLI` keeps its name, and so does what a read shows of a stored source.
|
|
209
|
+
*/
|
|
210
|
+
const RESERVED_LEADING_WORDS = new Set(['system', 'developer', 'assistant', 'renor']);
|
|
211
|
+
const SOURCE_IDENTITY_SEPARATORS = /[ ._-]/g;
|
|
212
|
+
/** The digits and signs a typed name can stand in for a letter with (`SYST3M`, `0wner`). `1` is tried as both i and l. */
|
|
213
|
+
const DIGIT_LOOKALIKES = { '0': 'o', '3': 'e', '4': 'a', '5': 's', '7': 't', '@': 'a', '$': 's' };
|
|
214
|
+
const EXACT_RULE = { leadingWords: false, digits: false };
|
|
215
|
+
const CLIENT_RULE = { leadingWords: true, digits: true };
|
|
216
|
+
/** The lower-case forms a name is judged in: NFKC, accents dropped, and (with `digits`) its digit look-alikes read as letters. */
|
|
217
|
+
function judgedForms(source, digits) {
|
|
218
|
+
const base = source.normalize('NFKC').normalize('NFD').replace(/\p{M}+/gu, '').toLowerCase();
|
|
219
|
+
if (!digits)
|
|
220
|
+
return [base];
|
|
221
|
+
const letters = base.replace(/[03457@$]/g, ch => DIGIT_LOOKALIKES[ch]);
|
|
222
|
+
return [...new Set([letters.replace(/1/g, 'i'), letters.replace(/1/g, 'l')])];
|
|
223
|
+
}
|
|
224
|
+
/** The reserved identity a writer spelling claims under `rule`: the whole name, its `role:` prefix, or its first word. */
|
|
225
|
+
function reservedSourceIdentity(source, rule) {
|
|
226
|
+
for (const lower of judgedForms(source, rule.digits)) {
|
|
227
|
+
const identity = lower.replace(SOURCE_IDENTITY_SEPARATORS, '');
|
|
228
|
+
if (RESERVED_SOURCE_IDENTITIES.has(identity))
|
|
229
|
+
return identity;
|
|
230
|
+
const colon = lower.indexOf(':');
|
|
231
|
+
const authorityPrefix = colon < 0 ? null : lower.slice(0, colon).replace(SOURCE_IDENTITY_SEPARATORS, '');
|
|
232
|
+
if (authorityPrefix !== null && RESERVED_SOURCE_IDENTITIES.has(authorityPrefix))
|
|
233
|
+
return authorityPrefix;
|
|
234
|
+
if (!rule.leadingWords)
|
|
235
|
+
continue;
|
|
236
|
+
const firstWord = lower.split(/[^\p{L}\p{N}]+/u).find(word => word !== '');
|
|
237
|
+
if (firstWord !== undefined && RESERVED_LEADING_WORDS.has(firstWord))
|
|
238
|
+
return firstWord;
|
|
239
|
+
}
|
|
240
|
+
return null;
|
|
241
|
+
}
|
|
242
|
+
/** The reserved identity under any spelling a write persists: the name as given, the fact's cleanSource and the note's headerSource. */
|
|
243
|
+
function reservedUnder(source, rule) {
|
|
244
|
+
for (const form of [source, cleanSource(source), headerSource(source)]) {
|
|
245
|
+
const reserved = reservedSourceIdentity(form, rule);
|
|
246
|
+
if (reserved)
|
|
247
|
+
return reserved;
|
|
248
|
+
}
|
|
249
|
+
return null;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* The reserved identity a name a client chose would claim: a typed `source`, a User-Agent product, or a self-hosted OAuth
|
|
253
|
+
* client_name (the client registers it). The exact rule, plus a reserved first word and digit look-alikes.
|
|
254
|
+
*/
|
|
255
|
+
function reservedCallerSource(source) {
|
|
256
|
+
return reservedUnder(source, CLIENT_RULE);
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* The reserved identity a name claims exactly (whole name or `role:` prefix, fullwidth and accents folded): the rule for a
|
|
260
|
+
* gateway key's name, which the owner chose, and for what a read shows of a stored source. Paths, URLs, truncation and
|
|
261
|
+
* punctuation folding cannot manufacture one past this.
|
|
262
|
+
*/
|
|
263
|
+
function reservedSourceIn(source) {
|
|
264
|
+
return reservedUnder(source, EXACT_RULE);
|
|
265
|
+
}
|
|
266
|
+
/** Whether a stored source names a reserved identity: one saved before the rule, or written into a file by hand. */
|
|
267
|
+
function isReservedSource(source) {
|
|
268
|
+
return typeof source === 'string' && source !== '' && reservedSourceIn(source) !== null;
|
|
269
|
+
}
|
|
194
270
|
function authorMarker(author, reader) {
|
|
195
271
|
const fromOther = typeof author === 'string' && author !== '' && author !== reader;
|
|
196
272
|
return {
|
|
197
273
|
...(fromOther ? { source: author } : {}),
|
|
198
|
-
...(isImportSource(author) ? { untrusted: true } : {}),
|
|
274
|
+
...(isImportSource(author) || isReservedSource(author) ? { untrusted: true } : {}),
|
|
199
275
|
};
|
|
200
276
|
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Short-lived status kept from misleading later readers (RENOR_STATUS_FRESHNESS, default OFF).
|
|
4
|
+
*
|
|
5
|
+
* Agents save status as facts ("PR 648 is not merged", "main is at 3f9a1c2", "12 tests failing") that are wrong within
|
|
6
|
+
* hours. A fact is status-like when its subject/predicate/object names the state of a PR, commit, branch, build or
|
|
7
|
+
* count. Such a fact has a freshness window counted from its `validFrom`; once past it, a read labels it `stale`
|
|
8
|
+
* (`label`) or leaves it out (`hide`). Decisions, preferences and standing rules never match, so they are unaffected.
|
|
9
|
+
* Nothing is stored or rewritten: the window is derived from the saved text, so turning the switch on or off changes
|
|
10
|
+
* only what a read shows.
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.STATUS_FRESHNESS_FLAG = void 0;
|
|
14
|
+
exports.statusFreshnessMode = statusFreshnessMode;
|
|
15
|
+
exports.statusWindowHours = statusWindowHours;
|
|
16
|
+
exports.staleStatusOf = staleStatusOf;
|
|
17
|
+
exports.STATUS_FRESHNESS_FLAG = 'RENOR_STATUS_FRESHNESS';
|
|
18
|
+
function statusFreshnessMode(env = process.env) {
|
|
19
|
+
const raw = (env[exports.STATUS_FRESHNESS_FLAG] ?? '').trim().toLowerCase();
|
|
20
|
+
if (raw === 'hide')
|
|
21
|
+
return 'hide';
|
|
22
|
+
return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on' || raw === 'label' ? 'label' : 'off';
|
|
23
|
+
}
|
|
24
|
+
const HOUR = 3_600_000;
|
|
25
|
+
/** Standing wording wins over status wording: "always check whether the PR is merged" is a rule, not a status. */
|
|
26
|
+
const DURABLE = /\b(always|never|must|should|prefers?|policy|rule|convention|decid(?:ed|es|ion)|we (?:use|chose)|going forward|by default)\b/i;
|
|
27
|
+
const PATTERNS = [
|
|
28
|
+
// PR / issue / CI state
|
|
29
|
+
{ re: /\b(?:pr|pull request|issue)\s*#?\d+\b.*\b(?:open|closed|merged|not merged|unmerged|draft|approved|blocked|failing|passing|green|red|in review|waiting)\b/i, hours: 12 },
|
|
30
|
+
{ re: /\b(?:ci|build|checks?|pipeline|tests?)\b.*\b(?:failing|passing|green|red|broken|failed|flaky)\b/i, hours: 12 },
|
|
31
|
+
{ re: /\bnot (?:yet )?(?:merged|deployed|released|reviewed|pushed|shipped)\b/i, hours: 12 },
|
|
32
|
+
// commit / branch heads
|
|
33
|
+
{ re: /\b(?:main|master|develop|head|branch\s+\S+|origin\/\S+)\b.*\b(?:is at|points? to|at commit|head is|is on)\b.*\b[0-9a-f]{7,40}\b/i, hours: 6 },
|
|
34
|
+
{ re: /\b(?:latest|current|last) (?:commit|release|deploy(?:ment)?|version|build)\b/i, hours: 24 },
|
|
35
|
+
// counts that change
|
|
36
|
+
{ re: /\b\d+\s+(?:open\s+)?(?:prs?|pull requests|issues|tests?|failures|errors|todos?|tasks?)\s+(?:open|failing|passing|remaining|left|pending)\b/i, hours: 24 },
|
|
37
|
+
];
|
|
38
|
+
/** Hours the fact stays fresh, or null when it is not status-like. */
|
|
39
|
+
function statusWindowHours(fact) {
|
|
40
|
+
const text = `${fact.subject} ${fact.predicate.replace(/_/g, ' ')} ${fact.object}`;
|
|
41
|
+
if (DURABLE.test(text))
|
|
42
|
+
return null;
|
|
43
|
+
for (const { re, hours } of PATTERNS)
|
|
44
|
+
if (re.test(text))
|
|
45
|
+
return hours;
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
/** Set when a status-like fact is past its window at `now`. */
|
|
49
|
+
function staleStatusOf(fact, now = Date.now()) {
|
|
50
|
+
const hours = statusWindowHours(fact);
|
|
51
|
+
if (hours === null)
|
|
52
|
+
return null;
|
|
53
|
+
const saved = Date.parse(fact.validFrom);
|
|
54
|
+
if (Number.isNaN(saved))
|
|
55
|
+
return null;
|
|
56
|
+
const age = now - saved;
|
|
57
|
+
if (age <= hours * HOUR)
|
|
58
|
+
return null;
|
|
59
|
+
const ago = age >= 48 * HOUR ? `${Math.floor(age / (24 * HOUR))} days` : `${Math.floor(age / HOUR)} hours`;
|
|
60
|
+
return `status saved ${ago} ago; likely out of date, verify before relying on it`;
|
|
61
|
+
}
|
package/dist/memory/writer.js
CHANGED
|
@@ -13,6 +13,7 @@ const invisibleText_1 = require("./invisibleText");
|
|
|
13
13
|
const dailyNote_1 = require("./dailyNote");
|
|
14
14
|
const sourceName_1 = require("./sourceName");
|
|
15
15
|
const secretGuard_1 = require("../extraction/secretGuard");
|
|
16
|
+
const jevDecisionModel_1 = require("../extraction/jevDecisionModel");
|
|
16
17
|
const toolRefusal_1 = require("../toolRefusal");
|
|
17
18
|
const controls_1 = require("./controls");
|
|
18
19
|
const confinedWrite_1 = require("./confinedWrite");
|
|
@@ -149,8 +150,8 @@ async function appendDailyEntry(vaultPath, date, entry, memoryDir = 'Memory/Dail
|
|
|
149
150
|
if ((0, secretGuard_1.secretGuardEnabled)()) {
|
|
150
151
|
await (0, secretGuard_1.assertStorable)([
|
|
151
152
|
{ field: 'note text', text },
|
|
152
|
-
...sourceFormsForGuard(entry.source).map(form => ({ field: 'note source', text: form })),
|
|
153
|
-
]);
|
|
153
|
+
...sourceFormsForGuard(entry.source).map(form => ({ field: 'note source', text: form, nameOnly: true })),
|
|
154
|
+
], 'save', (0, jevDecisionModel_1.saveDecisionModel)());
|
|
154
155
|
}
|
|
155
156
|
const dir = path_1.default.join(vaultPath, memoryDir);
|
|
156
157
|
const filePath = path_1.default.join(dir, `${date}.md`);
|
|
@@ -678,6 +678,9 @@ async function collectMemories(ctx, keep, mayMatch) {
|
|
|
678
678
|
const newest = keep ? dated : [...dated].sort((x, y) => y.at.localeCompare(x.at)).slice(0, MAX_ENTRIES_PER_DAY);
|
|
679
679
|
if (newest.length < blocks.length)
|
|
680
680
|
truncated = true;
|
|
681
|
+
// An entry past the cap was still read, so it is live: a retirement of it that never completed is not listed.
|
|
682
|
+
for (const b of blocks)
|
|
683
|
+
seen.add(b.id);
|
|
681
684
|
for (const { b, at } of newest) {
|
|
682
685
|
add({
|
|
683
686
|
id: b.id, kind: 'entry', path: rel,
|
|
@@ -1066,7 +1069,8 @@ async function readMemory(ctx, id) {
|
|
|
1066
1069
|
return {
|
|
1067
1070
|
ok: true,
|
|
1068
1071
|
memory: {
|
|
1069
|
-
|
|
1072
|
+
// A note not indexed yet is dated by its file, as an entry is above.
|
|
1073
|
+
id, kind: 'note', path: file.rel, title: row?.title ?? parsed.title, at: row?.modified_at ?? (await promises_1.default.stat(file.real)).mtime.toISOString(),
|
|
1070
1074
|
...(dated ? { session_date: dated } : {}),
|
|
1071
1075
|
source: typeof parsed.frontmatter.source === 'string' ? parsed.frontmatter.source : null,
|
|
1072
1076
|
tags: tagsOf(parsed.frontmatter),
|
|
@@ -1407,7 +1411,8 @@ async function bytesBelow(dir) {
|
|
|
1407
1411
|
* (`retired:<hex>`), or `all` for every retired note and entry. With `all`, `only` (retirement ids, as the owner saw them)
|
|
1408
1412
|
* limits the deletion to those, so a retirement made after the owner's list was read is not deleted unseen. A live memory
|
|
1409
1413
|
* has no retirement id, so it can never be deleted in one step: it is retired first. A retired fact is kept in the fact
|
|
1410
|
-
* ledger, which is replayed to rebuild the index
|
|
1414
|
+
* ledger, which is replayed to rebuild the index. A bare `all` keeps retired facts; a fact is erased only when its
|
|
1415
|
+
* `fact:` id is targeted directly or included in `only`.
|
|
1411
1416
|
*
|
|
1412
1417
|
* What goes: a note's stored copy (so its bytes stop counting toward storage), an entry's text in the journal, every
|
|
1413
1418
|
* journal record of the memory's earlier retirements that carried its text, the retire, restore and note events that
|
|
@@ -1421,10 +1426,22 @@ async function bytesBelow(dir) {
|
|
|
1421
1426
|
* deletion (from then on nothing lists or restores it); a stored copy left behind by a crash after it is removed by the
|
|
1422
1427
|
* next retire, restore or delete (removePurgedCopies), and until then it counts toward storage but nothing reads it.
|
|
1423
1428
|
*/
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1429
|
+
/**
|
|
1430
|
+
* A fact erased for good, and the evidence cached from the ledger that held it: the evidence pass reads the ledger whole, so
|
|
1431
|
+
* its extractions can carry the erased value (the same as a daily log's, below). Dropped after the rewrite, so an extraction
|
|
1432
|
+
* that follows reads the tombstone; a retry of an erasure that already happened drops them again.
|
|
1433
|
+
*/
|
|
1434
|
+
async function eraseFact(ctx, id) {
|
|
1435
|
+
const receipt = await (0, lifecycle_1.eraseRetiredFact)(ctx.db, ctx.memoryDir, ctx.ledgerRelPath, id, ctx.indexOptions.vectorIndex);
|
|
1436
|
+
if (ctx.evidenceCachePath) {
|
|
1437
|
+
const ledger = (0, lifecycle_1.resolveLedgerPath)(ctx.memoryDir, ctx.ledgerRelPath);
|
|
1438
|
+
(0, evidenceCache_1.dropEvidenceForStems)(ctx.evidenceCachePath, [ledger, path_1.default.relative(path_1.default.resolve(ctx.memoryDir), ledger)]);
|
|
1427
1439
|
}
|
|
1440
|
+
return receipt;
|
|
1441
|
+
}
|
|
1442
|
+
async function purgeRetired(ctx, target, only) {
|
|
1443
|
+
if ((0, lifecycle_1.factEventId)(target) !== null)
|
|
1444
|
+
return eraseFact(ctx, target);
|
|
1428
1445
|
if (target !== 'all' && !target.startsWith('retired:')) {
|
|
1429
1446
|
throw (0, toolRefusal_1.refusal)('invalid_argument', 'pass the id of a retired memory from memory_map.retired (retired:<id>) or "all"; a live memory is retired first');
|
|
1430
1447
|
}
|
|
@@ -1436,18 +1453,27 @@ async function purgeRetired(ctx, target, only) {
|
|
|
1436
1453
|
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
1437
1454
|
const journal = await readJournal(ctx);
|
|
1438
1455
|
const live = (await collectMemories(ctx)).seen;
|
|
1439
|
-
const completed =
|
|
1456
|
+
const completed = await completedRetirements(ctx, live);
|
|
1440
1457
|
const listed = only === undefined ? null : new Set(only);
|
|
1441
1458
|
const chosen = target === 'all'
|
|
1442
|
-
? completed.filter((r) => listed === null
|
|
1459
|
+
? completed.filter((r) => listed === null ? r.memory.kind !== 'fact' : listed.has(r.memory.id))
|
|
1443
1460
|
: completed.filter((r) => r.memory.id === target);
|
|
1444
1461
|
if (target !== 'all' && chosen.length === 0)
|
|
1445
1462
|
throw (0, toolRefusal_1.refusal)('memory_not_found', 'no retired note or entry with that id');
|
|
1446
1463
|
const at = new Date().toISOString();
|
|
1447
1464
|
if (chosen.length === 0)
|
|
1448
1465
|
return { ok: true, action: 'deleted', ids: [], freedBytes: 0, at };
|
|
1449
|
-
const
|
|
1450
|
-
const
|
|
1466
|
+
const facts = chosen.filter((r) => r.memory.kind === 'fact');
|
|
1467
|
+
const files = chosen.filter((r) => r.memory.kind !== 'fact');
|
|
1468
|
+
const erasedFacts = [];
|
|
1469
|
+
for (const fact of facts) {
|
|
1470
|
+
const erased = await eraseFact(ctx, fact.memory.id);
|
|
1471
|
+
erasedFacts.push(...erased.ids);
|
|
1472
|
+
}
|
|
1473
|
+
if (files.length === 0)
|
|
1474
|
+
return { ok: true, action: 'deleted', ids: erasedFacts, freedBytes: 0, at };
|
|
1475
|
+
const rids = new Set(files.map((r) => r.memory.id));
|
|
1476
|
+
const memoryIds = new Set(files.map((r) => r.memory.memoryId));
|
|
1451
1477
|
const byRid = new Map(activeRetired(journal).map((r) => [r.rid, r]));
|
|
1452
1478
|
let freedBytes = 0;
|
|
1453
1479
|
for (const rid of rids) {
|
|
@@ -1458,14 +1484,14 @@ async function purgeRetired(ctx, target, only) {
|
|
|
1458
1484
|
// 1. Events, cached evidence, import-journal entries and replacement records that carry its text or path. Idempotent, so a retry after a
|
|
1459
1485
|
// crash repeats them harmlessly.
|
|
1460
1486
|
const stems = new Set();
|
|
1461
|
-
await (0, forgetImported_1.forgetImportedNotes)(ctx.memoryDir, new Set(
|
|
1462
|
-
await (0, replacedJournal_1.forgetReplacementsOf)(path_1.default.join(rootReal, exports.RETIRED_DIR), new Set(
|
|
1487
|
+
await (0, forgetImported_1.forgetImportedNotes)(ctx.memoryDir, new Set(files.filter((r) => r.memory.kind === 'note').map((r) => r.memory.path)));
|
|
1488
|
+
await (0, replacedJournal_1.forgetReplacementsOf)(path_1.default.join(rootReal, exports.RETIRED_DIR), new Set(files.map((r) => r.memory.memoryId).filter((id) => typeof id === 'string')));
|
|
1463
1489
|
ctx.db.transaction(() => {
|
|
1464
1490
|
const named = ctx.db.prepare(`DELETE FROM write_events WHERE action IN ('retire', 'restore') AND json_valid(summary) AND json_extract(summary, '$.id') = ?`);
|
|
1465
1491
|
const noted = ctx.db.prepare(`DELETE FROM write_events WHERE action = 'note' AND note_path IN (?, ?) AND summary = ?`);
|
|
1466
1492
|
for (const memoryId of memoryIds)
|
|
1467
1493
|
named.run(memoryId);
|
|
1468
|
-
for (const r of
|
|
1494
|
+
for (const r of files) {
|
|
1469
1495
|
const rel = r.memory.path;
|
|
1470
1496
|
stems.add(rel);
|
|
1471
1497
|
stems.add(path_1.default.resolve(ctx.memoryDir, ...rel.split('/')));
|
|
@@ -1502,7 +1528,7 @@ async function purgeRetired(ctx, target, only) {
|
|
|
1502
1528
|
.filter(Boolean).join(' and ');
|
|
1503
1529
|
ctx.db.prepare('INSERT INTO write_events (note_path, action, summary, created_at, author) VALUES (?, ?, ?, ?, ?)')
|
|
1504
1530
|
.run('_retired_', 'delete', `deleted permanently: ${counted}`, at, ctx.source ?? null);
|
|
1505
|
-
return { ok: true, action: 'deleted', ids: [...rids], freedBytes, at };
|
|
1531
|
+
return { ok: true, action: 'deleted', ids: [...erasedFacts, ...rids], freedBytes, at };
|
|
1506
1532
|
});
|
|
1507
1533
|
}
|
|
1508
1534
|
/** Every entry of one daily log as stored (identical entries are one, as everywhere), in file order. */
|
|
@@ -64,7 +64,7 @@ function sourceForLabel(source) {
|
|
|
64
64
|
*/
|
|
65
65
|
function labelOf(replaced, correction) {
|
|
66
66
|
const saver = sourceForLabel(correction.source);
|
|
67
|
-
const label = { by: correction.by, at: correction.at, ...(saver ? { source: saver } : {}) };
|
|
67
|
+
const label = { by: correction.by, at: correction.at, ...(saver ? { source: saver } : {}), ...((0, sourceName_1.isReservedSource)(saver) ? { untrusted: true } : {}) };
|
|
68
68
|
if (replaced.oldSource === undefined)
|
|
69
69
|
return label;
|
|
70
70
|
const from = sourceForLabel(replaced.oldSource);
|
|
@@ -232,10 +232,15 @@ async function recordReplacement(ctx, old, by, source) {
|
|
|
232
232
|
...(oldSource !== undefined ? { oldSource } : {}),
|
|
233
233
|
};
|
|
234
234
|
const dir = await (0, memoryMap_1.ensureRealDir)(await promises_1.default.realpath(ctx.memoryDir), memoryMap_1.RETIRED_DIR);
|
|
235
|
-
|
|
235
|
+
// O_NONBLOCK and the single-name check: the same rules as the retirement journal (appendJournal), so neither a FIFO nor a
|
|
236
|
+
// hard link under this name is written through.
|
|
237
|
+
const handle = await promises_1.default.open(path_1.default.join(dir, path_1.default.basename(REPLACED_FILE)), promises_1.default.constants.O_RDWR | promises_1.default.constants.O_APPEND | promises_1.default.constants.O_CREAT | promises_1.default.constants.O_NONBLOCK | memoryMap_1.NOFOLLOW, 0o600);
|
|
236
238
|
try {
|
|
239
|
+
const stat = await handle.stat();
|
|
240
|
+
if (!stat.isFile() || stat.nlink !== 1)
|
|
241
|
+
throw new Error(`refusing to write ${JSON.stringify(replacedJournal_1.REPLACED_FILE_NAME)}: it is not a regular file with a single name`);
|
|
237
242
|
// A crash can leave a torn last line; start on a fresh line so this record is never glued to it.
|
|
238
|
-
const size =
|
|
243
|
+
const size = stat.size;
|
|
239
244
|
let prefix = '';
|
|
240
245
|
if (size > 0) {
|
|
241
246
|
const last = Buffer.alloc(1);
|
package/dist/mvp/agentMemory.js
CHANGED
|
@@ -3,13 +3,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.STATEMENT_RELEVANCE_FLOOR = exports.NEWEST_ANCHOR_RELEVANCE_FLOOR = exports.NEWEST_SESSION_WEAK_STATEMENTS = exports.NEWEST_SESSION_STATEMENT_WEIGHT = exports.NEWEST_SESSION_STATEMENT_FLOOR = exports.NEWEST_SESSION_PIECES = exports.NEWEST_SESSION_SHARE = exports.FACTS_BUDGET_SHARE = exports.CONTEXT_LEAN_ITEMS_FLAG = exports.PERMISSION_RULE_LIMIT = exports.GOVERNANCE_PREDICATES = exports.TRUNCATION_MARKER = exports.MAX_FACT_OBJECT_LANE_SHARE = exports.FTS_STOP_WORDS = void 0;
|
|
6
|
+
exports.STATEMENT_RELEVANCE_FLOOR = exports.NEWEST_ANCHOR_RELEVANCE_FLOOR = exports.CONTEXT_SPLIT_FLAG = exports.NEWEST_SESSION_WEAK_STATEMENTS = exports.NEWEST_SESSION_STATEMENT_WEIGHT = exports.NEWEST_SESSION_STATEMENT_FLOOR = exports.NEWEST_SESSION_PIECES = exports.NEWEST_SESSION_SHARE = exports.FACTS_BUDGET_SHARE = exports.CONTEXT_LEAN_ITEMS_FLAG = exports.PERMISSION_RULE_LIMIT = exports.GOVERNANCE_PREDICATES = exports.TRUNCATION_MARKER = exports.MAX_FACT_OBJECT_LANE_SHARE = exports.FTS_STOP_WORDS = void 0;
|
|
7
7
|
exports.factQueryTerms = factQueryTerms;
|
|
8
8
|
exports.joinedWords = joinedWords;
|
|
9
9
|
exports.wordStems = wordStems;
|
|
10
10
|
exports.scoreFactRelevance = scoreFactRelevance;
|
|
11
11
|
exports.rankCandidateFacts = rankCandidateFacts;
|
|
12
12
|
exports.capFactObject = capFactObject;
|
|
13
|
+
exports.keywordOrAll = keywordOrAll;
|
|
14
|
+
exports.keywordOrMatch = keywordOrMatch;
|
|
15
|
+
exports.searchTextFallbackEnabled = searchTextFallbackEnabled;
|
|
16
|
+
exports.keywordVariants = keywordVariants;
|
|
13
17
|
exports.isGovernancePredicate = isGovernancePredicate;
|
|
14
18
|
exports.permissionQuestion = permissionQuestion;
|
|
15
19
|
exports.permissionMeaning = permissionMeaning;
|
|
@@ -17,9 +21,12 @@ exports.currentLifecycleFacts = currentLifecycleFacts;
|
|
|
17
21
|
exports.nameOnFile = nameOnFile;
|
|
18
22
|
exports.factLaneRoom = factLaneRoom;
|
|
19
23
|
exports.searchRelevance = searchRelevance;
|
|
24
|
+
exports.scoreForReply = scoreForReply;
|
|
25
|
+
exports.headingForReply = headingForReply;
|
|
20
26
|
exports.contextLeanItemsEnabled = contextLeanItemsEnabled;
|
|
21
27
|
exports.leanMemory = leanMemory;
|
|
22
28
|
exports.dedupeMemories = dedupeMemories;
|
|
29
|
+
exports.budgetSplit = budgetSplit;
|
|
23
30
|
exports.replyTokens = replyTokens;
|
|
24
31
|
exports.matchBudgetOrder = matchBudgetOrder;
|
|
25
32
|
exports.newestMatchAbove = newestMatchAbove;
|
|
@@ -32,9 +39,11 @@ const path_1 = __importDefault(require("path"));
|
|
|
32
39
|
const tokenizer_1 = require("../indexer/tokenizer");
|
|
33
40
|
const sanitize_1 = require("./sanitize");
|
|
34
41
|
const hiddenSpans_1 = require("./hiddenSpans");
|
|
42
|
+
const rankingTrace_1 = require("../retrieval/rankingTrace");
|
|
35
43
|
const engine_1 = require("./engine");
|
|
36
44
|
const digests_1 = require("../digests/digests");
|
|
37
45
|
const scope_1 = require("./scope");
|
|
46
|
+
const contextDistinct_1 = require("./contextDistinct");
|
|
38
47
|
const factMarkers_1 = require("./factMarkers");
|
|
39
48
|
const sourceName_1 = require("../memory/sourceName");
|
|
40
49
|
const orchestrator_1 = require("../retrieval/orchestrator");
|
|
@@ -50,30 +59,8 @@ const readCache_1 = require("../db/readCache");
|
|
|
50
59
|
const statements_1 = require("../db/statements");
|
|
51
60
|
const contextRelevance_1 = require("./contextRelevance");
|
|
52
61
|
const replacements_1 = require("../memorymap/replacements");
|
|
53
|
-
|
|
54
|
-
const
|
|
55
|
-
/**
|
|
56
|
-
* Map of fact identity -> source path, resolved ONLY when read-time honesty is
|
|
57
|
-
* armed. Kept out of `currentLifecycleFacts` so the default response contract
|
|
58
|
-
* (and every existing caller) is unchanged when the flag is off.
|
|
59
|
-
*/
|
|
60
|
-
function factSourcePaths(db, scopeFilter) {
|
|
61
|
-
const inScope = scopeFilter === undefined ? undefined : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scopeFilter), 'f.scope');
|
|
62
|
-
const rows = db
|
|
63
|
-
.prepare(`SELECT e.canonical_name AS subject, f.predicate AS predicate,
|
|
64
|
-
COALESCE(f.scope, 'global') AS scope, f.source_path AS sourcePath
|
|
65
|
-
FROM facts f
|
|
66
|
-
JOIN entities e ON e.id = f.subject_entity_id
|
|
67
|
-
WHERE f.valid_to IS NULL
|
|
68
|
-
AND f.source_path = '_mcp_'${inScope ? `
|
|
69
|
-
AND ${inScope.sql}` : ''}`)
|
|
70
|
-
.all(...(inScope?.params ?? []));
|
|
71
|
-
const out = new Map();
|
|
72
|
-
for (const r of rows) {
|
|
73
|
-
out.set(`${r.subject}\u0000${r.predicate}\u0000${r.scope ?? 'global'}`, r.sourcePath);
|
|
74
|
-
}
|
|
75
|
-
return out;
|
|
76
|
-
}
|
|
62
|
+
const callPhases_1 = require("../callPhases");
|
|
63
|
+
const statusFacts_1 = require("../memory/statusFacts");
|
|
77
64
|
exports.FTS_STOP_WORDS = new Set([
|
|
78
65
|
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'what', 'which', 'who', 'where', 'when',
|
|
79
66
|
'why', 'how', 'do', 'does', 'did', 'to', 'for', 'of', 'in', 'on', 'at', 'with', 'and',
|
|
@@ -352,6 +339,23 @@ function capFactObject(object, maxTokens) {
|
|
|
352
339
|
}
|
|
353
340
|
/** Whether a listed fact's object is `value`, or the prefix of it the lane cut (a cut object carries TRUNCATION_MARKER). */
|
|
354
341
|
const objectIs = (shown, value) => shown === value || (shown.includes('…') && value.startsWith(shown.split('…')[0]));
|
|
342
|
+
/**
|
|
343
|
+
* H11b (RENOR_KEYWORD_OR_ALL=1, default off): the keyword lane issues one FTS expression, the question's content words joined
|
|
344
|
+
* by OR (BM25 ranks rare words highest), instead of the variants. Unset, nothing changes. Pre-registered in
|
|
345
|
+
* data/renor-goal-better-temporal-2/prereg/H11b-H18-H12.md.
|
|
346
|
+
*/
|
|
347
|
+
function keywordOrAll(env = process.env) {
|
|
348
|
+
return (env.RENOR_KEYWORD_OR_ALL ?? '').trim() === '1';
|
|
349
|
+
}
|
|
350
|
+
/** The FTS expression of H11b: each content word quoted, joined by OR. */
|
|
351
|
+
function keywordOrMatch(query) {
|
|
352
|
+
const terms = [...new Set((0, factTerms_1.words)(query).filter(word => !exports.FTS_STOP_WORDS.has(word)))];
|
|
353
|
+
return terms.map(term => `"${term.replace(/"/g, '""')}"`).join(' OR ');
|
|
354
|
+
}
|
|
355
|
+
/** H12 (RENOR_SEARCH_TEXT_FALLBACK=1, default off): search_text falls back to an OR over the words when the exact phrase is empty. */
|
|
356
|
+
function searchTextFallbackEnabled(env = process.env) {
|
|
357
|
+
return (env.RENOR_SEARCH_TEXT_FALLBACK ?? '').trim() === '1';
|
|
358
|
+
}
|
|
355
359
|
function keywordVariants(query) {
|
|
356
360
|
const compact = (0, factTerms_1.words)(query).filter(word => !exports.FTS_STOP_WORDS.has(word));
|
|
357
361
|
const variants = [query.trim()];
|
|
@@ -654,8 +658,30 @@ function searchRelevance(db, engine, memoryDir, read, probe, hits, limit) {
|
|
|
654
658
|
}
|
|
655
659
|
/** A note the import tool wrote (importing/hosted.ts: `Imported/<label>/…`), never one of the engine's own notes. */
|
|
656
660
|
const IMPORTED_PATH = /^Imported\//;
|
|
657
|
-
|
|
658
|
-
|
|
661
|
+
/**
|
|
662
|
+
* A score as a reply lists it: six decimals, whatever else is on the page. Tools tell agents to judge hits by score, so the
|
|
663
|
+
* rounding must not tie hits the ranking told apart: four decimals did (0.50004 and 0.50003 are both 0.5), six tie only
|
|
664
|
+
* scores closer than 5e-7, and no pair on the 50 search_memory pages of the agent-workflow eval. A hit shows the same score on
|
|
665
|
+
* every page.
|
|
666
|
+
*/
|
|
667
|
+
function scoreForReply(score) {
|
|
668
|
+
return Math.round(score * 1e6) / 1e6;
|
|
669
|
+
}
|
|
670
|
+
/** A heading that only repeats the note's own name (a daily note's heading is its date) says nothing the path does not. */
|
|
671
|
+
function headingForReply(hit) {
|
|
672
|
+
if (typeof hit.notePath !== 'string' || !hit.headingPath)
|
|
673
|
+
return hit;
|
|
674
|
+
const file = hit.notePath.slice(hit.notePath.lastIndexOf('/') + 1).replace(/\.md$/i, '');
|
|
675
|
+
if (hit.headingPath !== file)
|
|
676
|
+
return hit;
|
|
677
|
+
const { headingPath: _headingPath, ...rest } = hit;
|
|
678
|
+
return rest;
|
|
679
|
+
}
|
|
680
|
+
function normalizeMemory(raw, fallbackReason, reader) {
|
|
681
|
+
if (!raw || typeof raw !== 'object')
|
|
682
|
+
return null;
|
|
683
|
+
const hit = raw;
|
|
684
|
+
if (typeof hit.text !== 'string' || !hit.text.trim())
|
|
659
685
|
return null;
|
|
660
686
|
const reason = typeof hit.reason === 'string' ? hit.reason : fallbackReason;
|
|
661
687
|
return {
|
|
@@ -675,6 +701,8 @@ function normalizeMemory(hit, fallbackReason, reader) {
|
|
|
675
701
|
// byte. The marker's tokens are budget tokens like any other item's (applyBudget `cost`).
|
|
676
702
|
...(reason.split('+').includes(orchestrator_1.SESSION_EXPAND_REASON) ? { matched: false } : {}),
|
|
677
703
|
...(IMPORTED_PATH.test(String(hit.notePath ?? '')) ? { origin: 'import' } : {}),
|
|
704
|
+
// A daily-log chunk an entry of which claims a reserved source (factMarkers.ts): marked here, before the budget counts it.
|
|
705
|
+
...((0, factMarkers_1.claimsReservedEntrySource)(hit.notePath, hit.text) ? { untrusted: true } : {}),
|
|
678
706
|
};
|
|
679
707
|
}
|
|
680
708
|
exports.CONTEXT_LEAN_ITEMS_FLAG = 'RENOR_CONTEXT_LEAN_ITEMS';
|
|
@@ -756,6 +784,22 @@ exports.NEWEST_SESSION_STATEMENT_WEIGHT = 2;
|
|
|
756
784
|
* tokens). A statement that shares two words, and every match retrieval scored, are not counted.
|
|
757
785
|
*/
|
|
758
786
|
exports.NEWEST_SESSION_WEAK_STATEMENTS = 1;
|
|
787
|
+
exports.CONTEXT_SPLIT_FLAG = 'RENOR_CONTEXT_SPLIT';
|
|
788
|
+
/**
|
|
789
|
+
* How get_context splits its budget between facts, the newest session and the older evidence (RENOR_CONTEXT_SPLIT, off by
|
|
790
|
+
* default; unset, or anything but `facts,recent,pieces`, gives the constants above). `facts` is the facts' share while memories
|
|
791
|
+
* wait and `recent` the newest session's share, both in (0, 1]; `pieces` is the most pieces of that session, an integer from
|
|
792
|
+
* 1 to 12. The older evidence takes what the two leave.
|
|
793
|
+
*/
|
|
794
|
+
function budgetSplit() {
|
|
795
|
+
const off = { facts: exports.FACTS_BUDGET_SHARE, recent: exports.NEWEST_SESSION_SHARE, pieces: exports.NEWEST_SESSION_PIECES };
|
|
796
|
+
const raw = process.env[exports.CONTEXT_SPLIT_FLAG];
|
|
797
|
+
if (!raw)
|
|
798
|
+
return off;
|
|
799
|
+
const [facts, recent, pieces] = raw.split(',').map(part => Number(part.trim()));
|
|
800
|
+
const share = (n) => Number.isFinite(n) && n > 0 && n <= 1;
|
|
801
|
+
return share(facts) && share(recent) && Number.isInteger(pieces) && pieces >= 1 && pieces <= 12 ? { facts, recent, pieces } : off;
|
|
802
|
+
}
|
|
759
803
|
/** The fewest of the question's content words the newest match above the bar must share with it (`newestMatchAbove`). */
|
|
760
804
|
exports.NEWEST_ANCHOR_RELEVANCE_FLOOR = 1;
|
|
761
805
|
/** Tokens of a reply as the engine counts them: o200k, the chunker's tokenizer, special-token syntax read as text.
|
|
@@ -983,7 +1027,7 @@ function applyBudget(facts, factEvents, memories, memoryEvents, allowance, cost,
|
|
|
983
1027
|
}
|
|
984
1028
|
function budgetWith(keepFirst, facts, factEvents, memories, memoryEvents, allowance, cost, { keep, closest, recent, mates = [], sessionStatements = [], relevance } = {}, ordered = false) {
|
|
985
1029
|
const added = (memory) => memories[memory].reason === orchestrator_1.SESSION_EXPAND_REASON;
|
|
986
|
-
const factShare = memories.some((_, i) => !added(i)) ? Math.floor(allowance *
|
|
1030
|
+
const factShare = memories.some((_, i) => !added(i)) ? Math.floor(allowance * budgetSplit().facts) : allowance;
|
|
987
1031
|
let used = 0;
|
|
988
1032
|
let held; // `keep`, once it has its room
|
|
989
1033
|
const fits = (item, limit) => {
|
|
@@ -1052,7 +1096,8 @@ function budgetWith(keepFirst, facts, factEvents, memories, memoryEvents, allowa
|
|
|
1052
1096
|
// while what it said elsewhere still fits, and the statement that decides the answer is usually one of the small ones.
|
|
1053
1097
|
const reserved = recent === undefined ? -1 : memories.indexOf(recent);
|
|
1054
1098
|
if (reserved >= 0) {
|
|
1055
|
-
const
|
|
1099
|
+
const split = budgetSplit();
|
|
1100
|
+
const room = Math.floor(allowance * split.recent);
|
|
1056
1101
|
const said = (item) => relevance?.(item) ?? 0;
|
|
1057
1102
|
const weight = (item) => (item.reason === orchestrator_1.SESSION_EXPAND_REASON ? exports.NEWEST_SESSION_STATEMENT_WEIGHT : 1);
|
|
1058
1103
|
const value = (item) => (weight(item) * said(item) * said(item)) / Math.max(1, cost(item));
|
|
@@ -1068,7 +1113,7 @@ function budgetWith(keepFirst, facts, factEvents, memories, memoryEvents, allowa
|
|
|
1068
1113
|
let taken = 0;
|
|
1069
1114
|
let weakTaken = 0;
|
|
1070
1115
|
for (const piece of pool) {
|
|
1071
|
-
if (taken >= (ordered ? Infinity :
|
|
1116
|
+
if (taken >= (ordered ? Infinity : split.pieces))
|
|
1072
1117
|
break;
|
|
1073
1118
|
const statement = piece.reason === orchestrator_1.SESSION_EXPAND_REASON;
|
|
1074
1119
|
const weak = !ordered && statement && said(piece) < 2;
|
|
@@ -1216,6 +1261,15 @@ function pinToCorrections(reply, correctionOf) {
|
|
|
1216
1261
|
if (cut)
|
|
1217
1262
|
reply.truncated = true;
|
|
1218
1263
|
}
|
|
1264
|
+
/**
|
|
1265
|
+
* What the semantic search's notices (mvp/engine.ts: a failed search, a rebuild) say keyword search still does, and what
|
|
1266
|
+
* get_context says instead when its keyword index is missing too: the reply never claims it in one sentence and denies
|
|
1267
|
+
* it in the next.
|
|
1268
|
+
*/
|
|
1269
|
+
const KEYWORD_STILL_ANSWERS = [
|
|
1270
|
+
['the saved facts and keyword search do not need it and still answer.', 'the saved facts still answer.'],
|
|
1271
|
+
['Keyword search works meanwhile.', 'The saved facts still answer.'],
|
|
1272
|
+
];
|
|
1219
1273
|
async function buildAgentContext(db, engine, memoryDir, options = {}) {
|
|
1220
1274
|
const query = (options.query ?? '').trim();
|
|
1221
1275
|
const scopeNorm = options.scope === undefined ? undefined : (0, scope_1.normalizeScope)(options.scope);
|
|
@@ -1256,8 +1310,9 @@ async function buildAgentContext(db, engine, memoryDir, options = {}) {
|
|
|
1256
1310
|
// The variants find the same chunks of the ledger: each chunk is parsed once for this read.
|
|
1257
1311
|
const ledgerRecords = new Map();
|
|
1258
1312
|
(0, readCache_1.inReadSnapshot)(db, () => {
|
|
1259
|
-
|
|
1260
|
-
|
|
1313
|
+
const orAll = keywordOrAll();
|
|
1314
|
+
for (const variant of orAll ? [query.trim()] : keywordVariants(query)) {
|
|
1315
|
+
const keyword = (0, engine_1.engineSearchText)(db, memoryDir, variant, 6, scopeNorm, options.includeHistory === true, options.ledgerRelPath, ledgerRecords, orAll ? keywordOrMatch(query) : undefined);
|
|
1261
1316
|
keywordAvailable = keywordAvailable || keyword.ok;
|
|
1262
1317
|
if (!keyword.ok) {
|
|
1263
1318
|
keywordFailed = true;
|
|
@@ -1272,17 +1327,26 @@ async function buildAgentContext(db, engine, memoryDir, options = {}) {
|
|
|
1272
1327
|
});
|
|
1273
1328
|
if (keywordFailed && !keywordAvailable) {
|
|
1274
1329
|
degraded.push('keyword');
|
|
1275
|
-
|
|
1276
|
-
|
|
1330
|
+
const semanticNotice = KEYWORD_STILL_ANSWERS.reduce((text, [claim, instead]) => text.replace(claim, instead), notice ?? '');
|
|
1331
|
+
notice = `${semanticNotice ? `${semanticNotice} ` : ''}Keyword search is unavailable: its index is missing, so nothing here was found by words.`;
|
|
1277
1332
|
}
|
|
1278
1333
|
}
|
|
1279
1334
|
// A permission question also looks for the standing facts by meaning, through the embedding path the search above
|
|
1280
1335
|
// used (and its query vector, when it ran), when that path is the calibrated (Vertex) model; otherwise, or when it
|
|
1281
1336
|
// fails, by words alone (permissionMeaning).
|
|
1282
1337
|
const meaning = query ? await permissionMeaning(db, engine, query, scopeNorm, probe.queryVector) : undefined;
|
|
1283
|
-
|
|
1338
|
+
// Status facts ("PR 648 is not merged") past their freshness window are labelled, or left out (RENOR_STATUS_FRESHNESS, off by default).
|
|
1339
|
+
const freshness = (0, statusFacts_1.statusFreshnessMode)(options.env);
|
|
1340
|
+
const readAt = Date.now();
|
|
1341
|
+
const staleOf = (fact) => freshness === 'off' ? null : (0, statusFacts_1.staleStatusOf)(fact, readAt);
|
|
1342
|
+
const facts = (0, readCache_1.inReadSnapshot)(db, () => currentFactMatches(db, query, scopeNorm, query ? 12 : 20, meaning))
|
|
1343
|
+
.filter(fact => freshness !== 'hide' || staleOf(fact) === null);
|
|
1344
|
+
(0, callPhases_1.markRetrieved)(); // the matches and the facts are in hand: what follows builds the reply (callPhases.ts)
|
|
1284
1345
|
// After the keyword matches, so a statement a keyword search also found is that match, not the added copy.
|
|
1285
|
-
const
|
|
1346
|
+
const isLedger = (0, ledgerPresentation_1.ledgerNoteMatcher)({ memoryDir, ledgerRelPath: options.ledgerRelPath ?? lifecycle_1.DEFAULT_LEDGER_REL });
|
|
1347
|
+
const deduplicated = dedupeMemories([...memories, ...added]);
|
|
1348
|
+
// RENOR_CONTEXT_DISTINCT (off by default): a memory that repeats the reply's facts or another memory is not listed.
|
|
1349
|
+
const merged = (0, contextDistinct_1.contextDistinctEnabled)(options.env) ? (0, contextDistinct_1.distinctMemories)(deduplicated, facts, isLedger) : deduplicated;
|
|
1286
1350
|
// A correction takes the best score of the memories it corrects and is cut to a share of the budget (`bindCorrections`), and
|
|
1287
1351
|
// the reply keeps them in pairs: a replaced memory is never listed without its correction, nor before it, whatever the
|
|
1288
1352
|
// budget cut and however the layout sorts (`pinToCorrections`, and `pinned` below). Awaited only when the workspace has a
|
|
@@ -1312,9 +1376,11 @@ async function buildAgentContext(db, engine, memoryDir, options = {}) {
|
|
|
1312
1376
|
const shaped = (fact) => {
|
|
1313
1377
|
const base = toAgentFact(fact, options.reader);
|
|
1314
1378
|
const line = fact.eventId === null ? undefined : disputes.get(fact.eventId);
|
|
1315
|
-
|
|
1379
|
+
const stale = staleOf(fact);
|
|
1380
|
+
const marked = stale === null ? base : { ...base, staleStatus: stale };
|
|
1381
|
+
return line === undefined ? marked : { ...marked, writerConflict: line };
|
|
1316
1382
|
};
|
|
1317
|
-
const factLaneBudget = hasMemories ? Math.floor(maxTokens *
|
|
1383
|
+
const factLaneBudget = hasMemories ? Math.floor(maxTokens * budgetSplit().facts) : maxTokens;
|
|
1318
1384
|
const maxFactObjectTokens = Math.max(32, Math.floor(factLaneBudget * exports.MAX_FACT_OBJECT_LANE_SHARE));
|
|
1319
1385
|
// The cap is a crowding rule: it applies only when the facts would not otherwise fit the lane, and never to the
|
|
1320
1386
|
// top-ranked fact — a long fact that is itself the answer is delivered whole when the lane has room for it.
|
|
@@ -1343,7 +1409,6 @@ async function buildAgentContext(db, engine, memoryDir, options = {}) {
|
|
|
1343
1409
|
// A memory of the ledger names no note and no heading (retrieval/ledgerPresentation.ts); the event in its heading is
|
|
1344
1410
|
// read first, so that a memory that is the event of a listed fact is not listed again. Its heading alone: its text is
|
|
1345
1411
|
// the user's words now, which may hold a line that looks like a record's heading.
|
|
1346
|
-
const isLedger = (0, ledgerPresentation_1.ledgerNoteMatcher)({ memoryDir, ledgerRelPath: options.ledgerRelPath ?? lifecycle_1.DEFAULT_LEDGER_REL });
|
|
1347
1412
|
const candidateFacts = shownFacts.map((fact, index) => capFact(fact, index));
|
|
1348
1413
|
const factEvents = shownFacts.map(fact => fact.eventId?.toLowerCase() ?? null);
|
|
1349
1414
|
// The agent-facing text: hidden spans labelled (hiddenSpans.ts). It is taken after every judgment above, which compares
|
|
@@ -1418,17 +1483,9 @@ async function buildAgentContext(db, engine, memoryDir, options = {}) {
|
|
|
1418
1483
|
const lean = !honestyLane && contextLeanItemsEnabled(options.env);
|
|
1419
1484
|
const listedTokens = (body) => replyTokens(JSON.stringify(lean ? { ...body, memories: body.memories.map(leanMemory) } : body));
|
|
1420
1485
|
const ledgerRel = options.ledgerRelPath ?? lifecycle_1.DEFAULT_LEDGER_REL;
|
|
1421
|
-
|
|
1422
|
-
//
|
|
1423
|
-
const factWithProvenance = (f) => {
|
|
1424
|
-
const sourcePath = paths.get(`${f.subject}\u0000${f.predicate}\u0000${f.scope ?? 'global'}`);
|
|
1425
|
-
return {
|
|
1426
|
-
...f,
|
|
1427
|
-
provenance: sourcePath && sourcePath !== LIFECYCLE_NOTE_PATH
|
|
1428
|
-
? (0, readHonesty_1.noteProvenance)(clientSafePath(memoryDir, sourcePath), f.validFrom)
|
|
1429
|
-
: (0, readHonesty_1.factProvenance)(f.validFrom, ledgerRel),
|
|
1430
|
-
};
|
|
1431
|
-
};
|
|
1486
|
+
// Each item the lane lists says where it came from and when it was last verified. A fact get_context lists is a
|
|
1487
|
+
// lifecycle fact (memory/factTerms.ts reads `source_path = '_mcp_'` only), so the ledger is its source.
|
|
1488
|
+
const factWithProvenance = (f) => ({ ...f, provenance: (0, readHonesty_1.factProvenance)(f.validFrom, ledgerRel) });
|
|
1432
1489
|
const memoryWithProvenance = (m) => ({
|
|
1433
1490
|
...m,
|
|
1434
1491
|
provenance: (0, readHonesty_1.noteProvenance)(m.notePath, m.session_date ?? null),
|
|
@@ -1718,6 +1775,8 @@ async function buildAgentContext(db, engine, memoryDir, options = {}) {
|
|
|
1718
1775
|
body = assemble(kept);
|
|
1719
1776
|
}
|
|
1720
1777
|
const reply = honestyLane ? withLane(kept, body) : body;
|
|
1778
|
+
// The budget's final decision, for the learning log (rankingTrace.ts): which of the candidates it considered it kept.
|
|
1779
|
+
(0, rankingTrace_1.currentRankingTrace)()?.budgeted(candidateMemories, kept.memories);
|
|
1721
1780
|
if (rankedBy === 'session_date') {
|
|
1722
1781
|
// Selection is finished, including the honesty lane's extra statements. Render those statements with their
|
|
1723
1782
|
// session: appending a newest update after an older match contradicts the order the reply promises. Summary
|