renor-engine 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +73 -0
- package/README.md +105 -2
- package/dist/BUILD.json +6 -0
- package/dist/api/apiAuth.js +43 -0
- package/dist/api/server.js +172 -0
- package/dist/buildIdentity.js +132 -0
- package/dist/cli-extractors.js +46 -0
- package/dist/cli.js +491 -0
- package/dist/config.js +65 -0
- package/dist/context/composer.js +320 -0
- package/dist/context/readHonesty.js +478 -0
- package/dist/context/rowStore.js +238 -0
- package/dist/conversation/formatter.js +81 -0
- package/dist/conversation/parser.js +189 -0
- package/dist/conversation/types.js +2 -0
- package/dist/conversation/writer.js +42 -0
- package/dist/db/client.js +147 -0
- package/dist/db/indexSalvage.js +81 -0
- package/dist/db/migrations.js +642 -0
- package/dist/db/readCache.js +109 -0
- package/dist/db/schema.js +5 -0
- package/dist/db/statements.js +89 -0
- package/dist/digests/digestCache.js +142 -0
- package/dist/digests/digests.js +597 -0
- package/dist/digests/worker.js +348 -0
- package/dist/embeddings/caching.js +337 -0
- package/dist/embeddings/coalescing.js +177 -0
- package/dist/embeddings/interactive.js +103 -0
- package/dist/embeddings/local.js +68 -0
- package/dist/embeddings/openai.js +105 -0
- package/dist/embeddings/rebuild.js +166 -0
- package/dist/embeddings/simdScan.js +258 -0
- package/dist/embeddings/space.js +21 -0
- package/dist/embeddings/types.js +14 -0
- package/dist/embeddings/vectorIndex.js +675 -0
- package/dist/embeddings/vertex.js +224 -0
- package/dist/eval/artifacts.js +42 -0
- package/dist/eval/cacheReport.js +139 -0
- package/dist/eval/comparison.js +182 -0
- package/dist/eval/context700-candidate.js +145 -0
- package/dist/eval/evalCache.js +525 -0
- package/dist/eval/hashEmbedding.js +35 -0
- package/dist/eval/longmemeval.js +56 -0
- package/dist/eval/metrics.js +181 -0
- package/dist/eval/renorbenchBaseline.js +486 -0
- package/dist/eval/renorbenchDocuments.js +134 -0
- package/dist/eval/renorbenchProduct.js +182 -0
- package/dist/eval/runLedger.js +279 -0
- package/dist/eval/runManifest.js +479 -0
- package/dist/eval/scoring.js +197 -0
- package/dist/evidence/attach.js +267 -0
- package/dist/evidence/background.js +147 -0
- package/dist/evidence/evidenceCache.js +182 -0
- package/dist/evidence/gate.js +37 -0
- package/dist/evidence/keyPool.js +85 -0
- package/dist/evidence/ollamaChat.js +84 -0
- package/dist/evidence/optin.js +165 -0
- package/dist/evidence/options.js +130 -0
- package/dist/evidence/secondPass.js +456 -0
- package/dist/extraction/classifyCapture.js +34 -0
- package/dist/extraction/decision-model.js +174 -0
- package/dist/extraction/eventExtractor.js +113 -0
- package/dist/extraction/extractionClient.js +122 -0
- package/dist/extraction/factExtractor.js +46 -0
- package/dist/extraction/factsWritePath.js +179 -0
- package/dist/extraction/preferenceExtractor.js +153 -0
- package/dist/extraction/secretGuard.js +176 -0
- package/dist/extraction/secretScanner.js +665 -0
- package/dist/extraction/typedFacts.js +224 -0
- package/dist/extraction/writePathAdjudication.js +110 -0
- package/dist/importing/apply.js +644 -0
- package/dist/importing/bundle-write.js +80 -0
- package/dist/importing/cli.js +407 -0
- package/dist/importing/existing.js +49 -0
- package/dist/importing/facts.js +123 -0
- package/dist/importing/forgetImported.js +79 -0
- package/dist/importing/hosted.js +467 -0
- package/dist/importing/index.js +29 -0
- package/dist/importing/journalLog.js +96 -0
- package/dist/importing/markdown.js +98 -0
- package/dist/importing/plan.js +206 -0
- package/dist/importing/secretScreen.js +113 -0
- package/dist/importing/serialize.js +188 -0
- package/dist/importing/source.js +463 -0
- package/dist/importing/types.js +6 -0
- package/dist/indexer/chunker.js +319 -0
- package/dist/indexer/dailyEntries.js +63 -0
- package/dist/indexer/hasher.js +10 -0
- package/dist/indexer/laneRuns.js +48 -0
- package/dist/indexer/o200k.js +481 -0
- package/dist/indexer/o200k.json +8 -0
- package/dist/indexer/outsideRoot.js +77 -0
- package/dist/indexer/parser.js +167 -0
- package/dist/indexer/pipeline.js +1168 -0
- package/dist/indexer/staleness.js +201 -0
- package/dist/indexer/tokenizer.js +77 -0
- package/dist/indexer/walker.js +184 -0
- package/dist/mcp/hive.js +88 -0
- package/dist/mcp/requestErrors.js +47 -0
- package/dist/mcp/server.js +396 -0
- package/dist/memory/agentLease.js +43 -0
- package/dist/memory/confinedWrite.js +315 -0
- package/dist/memory/controls.js +73 -0
- package/dist/memory/dailyNote.js +265 -0
- package/dist/memory/durableAppend.js +57 -0
- package/dist/memory/factConflicts.js +203 -0
- package/dist/memory/factTerms.js +321 -0
- package/dist/memory/hiveNameHeader.cjs +10 -0
- package/dist/memory/inventory.js +55 -0
- package/dist/memory/invisibleText.js +20 -0
- package/dist/memory/lifecycle.js +1756 -0
- package/dist/memory/limits.js +13 -0
- package/dist/memory/requests.js +183 -0
- package/dist/memory/sourceName.js +200 -0
- package/dist/memory/whatsNew.js +158 -0
- package/dist/memory/writer.js +175 -0
- package/dist/memory/writerConflicts.js +85 -0
- package/dist/memorymap/memoryMap.js +1568 -0
- package/dist/memorymap/replacedJournal.js +58 -0
- package/dist/memorymap/replacements.js +405 -0
- package/dist/mvp/activity.js +86 -0
- package/dist/mvp/agentMemory.js +1744 -0
- package/dist/mvp/auth.js +531 -0
- package/dist/mvp/cli.js +73 -0
- package/dist/mvp/client-store.js +462 -0
- package/dist/mvp/compileCache.js +249 -0
- package/dist/mvp/compileSeed.js +27 -0
- package/dist/mvp/config.js +288 -0
- package/dist/mvp/connectionStatus.js +80 -0
- package/dist/mvp/contextRelevance.js +413 -0
- package/dist/mvp/engine.js +1065 -0
- package/dist/mvp/exportAll.js +359 -0
- package/dist/mvp/factLanePack.js +77 -0
- package/dist/mvp/factMarkers.js +49 -0
- package/dist/mvp/factsLane.js +203 -0
- package/dist/mvp/hiddenSpans.js +585 -0
- package/dist/mvp/history.js +413 -0
- package/dist/mvp/hitDates.js +31 -0
- package/dist/mvp/hitFloor.js +36 -0
- package/dist/mvp/identity.js +18 -0
- package/dist/mvp/idle.js +113 -0
- package/dist/mvp/memoryDir.js +98 -0
- package/dist/mvp/posthogMcp.js +210 -0
- package/dist/mvp/readScope.js +104 -0
- package/dist/mvp/readUsefulness.js +222 -0
- package/dist/mvp/sanitize.js +48 -0
- package/dist/mvp/schemaRejections.js +91 -0
- package/dist/mvp/scope.js +127 -0
- package/dist/mvp/server.js +2089 -0
- package/dist/mvp/serverCard.js +29 -0
- package/dist/mvp/storedSize.js +139 -0
- package/dist/mvp/telemetry.js +441 -0
- package/dist/retrieval/adviceLines.js +195 -0
- package/dist/retrieval/adviceQuestion.js +44 -0
- package/dist/retrieval/conversationShape.js +44 -0
- package/dist/retrieval/dailyEntryTime.js +55 -0
- package/dist/retrieval/entityLookup.js +113 -0
- package/dist/retrieval/factsFirst.js +188 -0
- package/dist/retrieval/keyword.js +84 -0
- package/dist/retrieval/ledgerEventTime.js +51 -0
- package/dist/retrieval/ledgerLiveness.js +83 -0
- package/dist/retrieval/ledgerPresentation.js +161 -0
- package/dist/retrieval/noteIdentity.js +57 -0
- package/dist/retrieval/orchestrator.js +1086 -0
- package/dist/retrieval/queryHelpers.js +142 -0
- package/dist/retrieval/questionSpan.js +475 -0
- package/dist/retrieval/readerEvents.js +690 -0
- package/dist/retrieval/readerLayout.js +503 -0
- package/dist/retrieval/relativeTimeHints.js +113 -0
- package/dist/retrieval/semantic.js +67 -0
- package/dist/router/routerCache.js +106 -0
- package/dist/router/sessionRouter.js +726 -0
- package/dist/toolRefusal.js +120 -0
- package/dist/watcher/watcher.js +75 -0
- package/package.json +120 -4
|
@@ -0,0 +1,1744 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.STATEMENT_RELEVANCE_FLOOR = exports.NEWEST_ANCHOR_RELEVANCE_FLOOR = exports.NEWEST_SESSION_WEAK_STATEMENTS = exports.NEWEST_SESSION_STATEMENT_WEIGHT = exports.NEWEST_SESSION_STATEMENT_FLOOR = exports.NEWEST_SESSION_PIECES = exports.NEWEST_SESSION_SHARE = exports.FACTS_BUDGET_SHARE = exports.CONTEXT_LEAN_ITEMS_FLAG = exports.PERMISSION_RULE_LIMIT = exports.GOVERNANCE_PREDICATES = exports.TRUNCATION_MARKER = exports.MAX_FACT_OBJECT_LANE_SHARE = exports.FTS_STOP_WORDS = void 0;
|
|
7
|
+
exports.factQueryTerms = factQueryTerms;
|
|
8
|
+
exports.joinedWords = joinedWords;
|
|
9
|
+
exports.wordStems = wordStems;
|
|
10
|
+
exports.scoreFactRelevance = scoreFactRelevance;
|
|
11
|
+
exports.rankCandidateFacts = rankCandidateFacts;
|
|
12
|
+
exports.capFactObject = capFactObject;
|
|
13
|
+
exports.isGovernancePredicate = isGovernancePredicate;
|
|
14
|
+
exports.permissionQuestion = permissionQuestion;
|
|
15
|
+
exports.permissionMeaning = permissionMeaning;
|
|
16
|
+
exports.currentLifecycleFacts = currentLifecycleFacts;
|
|
17
|
+
exports.nameOnFile = nameOnFile;
|
|
18
|
+
exports.factLaneRoom = factLaneRoom;
|
|
19
|
+
exports.searchRelevance = searchRelevance;
|
|
20
|
+
exports.contextLeanItemsEnabled = contextLeanItemsEnabled;
|
|
21
|
+
exports.leanMemory = leanMemory;
|
|
22
|
+
exports.dedupeMemories = dedupeMemories;
|
|
23
|
+
exports.replyTokens = replyTokens;
|
|
24
|
+
exports.matchBudgetOrder = matchBudgetOrder;
|
|
25
|
+
exports.newestMatchAbove = newestMatchAbove;
|
|
26
|
+
exports.statementRelevance = statementRelevance;
|
|
27
|
+
exports.sessionStatementsFor = sessionStatementsFor;
|
|
28
|
+
exports.sessionMatesAbove = sessionMatesAbove;
|
|
29
|
+
exports.applyBudget = applyBudget;
|
|
30
|
+
exports.buildAgentContext = buildAgentContext;
|
|
31
|
+
const path_1 = __importDefault(require("path"));
|
|
32
|
+
const tokenizer_1 = require("../indexer/tokenizer");
|
|
33
|
+
const sanitize_1 = require("./sanitize");
|
|
34
|
+
const hiddenSpans_1 = require("./hiddenSpans");
|
|
35
|
+
const engine_1 = require("./engine");
|
|
36
|
+
const digests_1 = require("../digests/digests");
|
|
37
|
+
const scope_1 = require("./scope");
|
|
38
|
+
const factMarkers_1 = require("./factMarkers");
|
|
39
|
+
const sourceName_1 = require("../memory/sourceName");
|
|
40
|
+
const orchestrator_1 = require("../retrieval/orchestrator");
|
|
41
|
+
const attach_1 = require("../evidence/attach");
|
|
42
|
+
const readHonesty_1 = require("../context/readHonesty");
|
|
43
|
+
const lifecycle_1 = require("../memory/lifecycle");
|
|
44
|
+
const writerConflicts_1 = require("../memory/writerConflicts");
|
|
45
|
+
const ledgerLiveness_1 = require("../retrieval/ledgerLiveness");
|
|
46
|
+
const ledgerPresentation_1 = require("../retrieval/ledgerPresentation");
|
|
47
|
+
const factTerms_1 = require("../memory/factTerms");
|
|
48
|
+
const factConflicts_1 = require("../memory/factConflicts");
|
|
49
|
+
const readCache_1 = require("../db/readCache");
|
|
50
|
+
const statements_1 = require("../db/statements");
|
|
51
|
+
const contextRelevance_1 = require("./contextRelevance");
|
|
52
|
+
const replacements_1 = require("../memorymap/replacements");
|
|
53
|
+
/** Reserved synthetic note path owning lifecycle entities (mirrors lifecycle.ts). */
|
|
54
|
+
const LIFECYCLE_NOTE_PATH = '_mcp_';
|
|
55
|
+
/**
|
|
56
|
+
* Map of fact identity -> source path, resolved ONLY when read-time honesty is
|
|
57
|
+
* armed. Kept out of `currentLifecycleFacts` so the default response contract
|
|
58
|
+
* (and every existing caller) is unchanged when the flag is off.
|
|
59
|
+
*/
|
|
60
|
+
function factSourcePaths(db, scopeFilter) {
|
|
61
|
+
const inScope = scopeFilter === undefined ? undefined : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scopeFilter), 'f.scope');
|
|
62
|
+
const rows = db
|
|
63
|
+
.prepare(`SELECT e.canonical_name AS subject, f.predicate AS predicate,
|
|
64
|
+
COALESCE(f.scope, 'global') AS scope, f.source_path AS sourcePath
|
|
65
|
+
FROM facts f
|
|
66
|
+
JOIN entities e ON e.id = f.subject_entity_id
|
|
67
|
+
WHERE f.valid_to IS NULL
|
|
68
|
+
AND f.source_path = '_mcp_'${inScope ? `
|
|
69
|
+
AND ${inScope.sql}` : ''}`)
|
|
70
|
+
.all(...(inScope?.params ?? []));
|
|
71
|
+
const out = new Map();
|
|
72
|
+
for (const r of rows) {
|
|
73
|
+
out.set(`${r.subject}\u0000${r.predicate}\u0000${r.scope ?? 'global'}`, r.sourcePath);
|
|
74
|
+
}
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
exports.FTS_STOP_WORDS = new Set([
|
|
78
|
+
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'what', 'which', 'who', 'where', 'when',
|
|
79
|
+
'why', 'how', 'do', 'does', 'did', 'to', 'for', 'of', 'in', 'on', 'at', 'with', 'and',
|
|
80
|
+
'or', 'my', 'your', 'our', 'their', 'his', 'her', 'it', 'this', 'that', 'about',
|
|
81
|
+
// Two-letter words that never name a topic. Not every two-letter word: CI, DB, AI, Go or IP often are the topic.
|
|
82
|
+
'am', 'as', 'be', 'by', 'he', 'if', 'me', 'so', 'up', 'we',
|
|
83
|
+
]);
|
|
84
|
+
/**
|
|
85
|
+
* The facts lane's stop words: FTS_STOP_WORDS and the other words that never say what a fact is about. Pronouns,
|
|
86
|
+
* auxiliaries and modals, negation and what a contraction leaves of it ("don't" is `don`), quantifiers, and the
|
|
87
|
+
* adverbs, prepositions and conjunctions of any question. A fact's object is often a sentence that has them, so a
|
|
88
|
+
* question found it by them: "Hi, how are you?" returned a finding that says "you can not run all the tests now" (live
|
|
89
|
+
* dogfood, 2026-10-03, D7). Words that can be the topic are not here: "us" (the country), "may" (the month), "won" (a
|
|
90
|
+
* deal), "down", "off", "out" and "over" (a state), and two-letter topics like CI or DB.
|
|
91
|
+
*/
|
|
92
|
+
const FACT_STOP_WORDS = new Set([
|
|
93
|
+
...exports.FTS_STOP_WORDS,
|
|
94
|
+
'you', 'him', 'she', 'they', 'them', 'its', 'mine', 'yours', 'ours', 'theirs', 'hers', 'these', 'those', 'whom', 'whose',
|
|
95
|
+
'myself', 'yourself', 'yourselves', 'himself', 'herself', 'itself', 'ourselves', 'themselves',
|
|
96
|
+
'can', 'could', 'will', 'would', 'shall', 'should', 'might', 'must', 'have', 'has', 'had', 'having', 'been', 'being', 'doing',
|
|
97
|
+
'not', 'no', 'nor', 'don', 'doesn', 'didn', 'isn', 'aren', 'wasn', 'weren', 'hasn', 'haven', 'hadn', 'couldn', 'wouldn',
|
|
98
|
+
'shouldn', 'mustn', 'needn', 'mightn', 'shan', 'ain', 'll', 're', 've',
|
|
99
|
+
'all', 'any', 'both', 'each', 'every', 'few', 'many', 'much', 'more', 'most', 'other', 'some', 'such', 'same', 'own',
|
|
100
|
+
'now', 'then', 'here', 'there', 'just', 'also', 'very', 'too', 'only', 'again', 'once', 'further',
|
|
101
|
+
'from', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'against', 'between',
|
|
102
|
+
'but', 'because', 'until', 'while', 'than',
|
|
103
|
+
]);
|
|
104
|
+
/**
|
|
105
|
+
* The words a fact is looked up by for `query`: its atoms (memory/factTerms.ts) without the facts lane's stop words, so
|
|
106
|
+
* "in", "you" or "not" never make a fact relevant. The stop words count only when the query has no other word.
|
|
107
|
+
*/
|
|
108
|
+
function factQueryTerms(query) {
|
|
109
|
+
const all = (0, factTerms_1.atoms)(query);
|
|
110
|
+
const content = all.filter(word => !FACT_STOP_WORDS.has(word));
|
|
111
|
+
return content.length > 0 ? content : all;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Two neighbouring words of `query` written as one, the way keys often are: "time zone" is `timezone`, "home lab" `homelab`,
|
|
115
|
+
* "Lane J" `lanej`. Single letters and digits following a word are joined as well so relation keys like "lanej" or "v2" match.
|
|
116
|
+
*/
|
|
117
|
+
function joinedWords(query) {
|
|
118
|
+
const sequence = (0, factTerms_1.atomSequence)(query);
|
|
119
|
+
const joined = [];
|
|
120
|
+
for (let i = 1; i < sequence.length; i++) {
|
|
121
|
+
// Never a stop word: "to do" is not the status `todo`.
|
|
122
|
+
if (!FACT_STOP_WORDS.has(sequence[i - 1]) && !FACT_STOP_WORDS.has(sequence[i]))
|
|
123
|
+
joined.push(sequence[i - 1] + sequence[i]);
|
|
124
|
+
}
|
|
125
|
+
const rawParts = query.toLowerCase().replace(/[^\p{L}\p{N}\p{M}_-]+/gu, ' ').split(/\s+/).map(w => w.trim()).filter(Boolean);
|
|
126
|
+
for (let i = 1; i < rawParts.length; i++) {
|
|
127
|
+
const prev = rawParts[i - 1];
|
|
128
|
+
const curr = rawParts[i];
|
|
129
|
+
// A trailing `s` is a possessive, not an initial: "John's plan" is not the key `johns`.
|
|
130
|
+
if (prev.length >= 2 && curr.length === 1 && curr !== 's' && !FACT_STOP_WORDS.has(prev) && !FACT_STOP_WORDS.has(curr) && curr !== 'a' && curr !== 'i') {
|
|
131
|
+
joined.push(prev + curr);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return Array.from(new Set(joined));
|
|
135
|
+
}
|
|
136
|
+
function wordStems(wordList) {
|
|
137
|
+
const out = new Set();
|
|
138
|
+
const irregularBases = { does: 'do', has: 'have' };
|
|
139
|
+
for (const word of wordList) {
|
|
140
|
+
out.add(word);
|
|
141
|
+
const irregular = irregularBases[word];
|
|
142
|
+
if (irregular) {
|
|
143
|
+
out.add(irregular);
|
|
144
|
+
}
|
|
145
|
+
else if (/^[a-z]+ies$/.test(word) && word.length >= 5) {
|
|
146
|
+
out.add(`${word.slice(0, -3)}y`);
|
|
147
|
+
}
|
|
148
|
+
else if (/^[a-z]+(ches|shes|xes|zes|oes|sses)$/.test(word)) {
|
|
149
|
+
out.add(word.slice(0, -2));
|
|
150
|
+
}
|
|
151
|
+
else if (/^[a-z]+s$/.test(word) && word.length >= 4 &&
|
|
152
|
+
!word.endsWith('ss') && !word.endsWith('us') && !word.endsWith('is')) {
|
|
153
|
+
out.add(word.slice(0, -1));
|
|
154
|
+
}
|
|
155
|
+
else if (/^[a-z]+ed$/.test(word) && word.length >= 5) {
|
|
156
|
+
out.add(word.slice(0, -2));
|
|
157
|
+
out.add(word.slice(0, -1));
|
|
158
|
+
}
|
|
159
|
+
else if (/^[a-z]+ing$/.test(word) && word.length >= 6) {
|
|
160
|
+
out.add(word.slice(0, -3));
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return Array.from(out);
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* The words a query looks facts up by, and the same with two neighbouring words joined: where ranking and scoring both
|
|
167
|
+
* start. Ranking scores every candidate against one query, so the words are worked out once, not once per fact.
|
|
168
|
+
*/
|
|
169
|
+
const FACT_SEARCH_TERMS_MAX = 8;
|
|
170
|
+
const factSearchTermsHeld = new Map();
|
|
171
|
+
function factSearchTerms(query) {
|
|
172
|
+
const held = factSearchTermsHeld.get(query);
|
|
173
|
+
if (held)
|
|
174
|
+
return held;
|
|
175
|
+
const terms = [...new Set([...factQueryTerms(query), ...joinedWords(query)])];
|
|
176
|
+
if (factSearchTermsHeld.size >= FACT_SEARCH_TERMS_MAX)
|
|
177
|
+
factSearchTermsHeld.delete(factSearchTermsHeld.keys().next().value);
|
|
178
|
+
factSearchTermsHeld.set(query, terms);
|
|
179
|
+
return terms;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Stems of one field of a fact (its subject, predicate, object or saved statement words). The text is the whole input.
|
|
183
|
+
* A read ranks a few dozen candidates and then scores each of them again, so a field is stemmed once per read, not twice
|
|
184
|
+
* per fact and word; a subject or predicate that many facts share costs one stemming for all of them. A few hundred
|
|
185
|
+
* fields are kept: the working set of one read, not of the workspace.
|
|
186
|
+
*/
|
|
187
|
+
const FIELD_STEMS_MAX = 256;
|
|
188
|
+
/** A field longer than this is stemmed again each time rather than held: one long object would take the room of dozens. */
|
|
189
|
+
const FIELD_STEMS_MAX_CHARS = 1024;
|
|
190
|
+
const fieldStems = new Map();
|
|
191
|
+
const statementWordStems = new Map();
|
|
192
|
+
function stemsOfField(text) {
|
|
193
|
+
const held = fieldStems.get(text);
|
|
194
|
+
if (held)
|
|
195
|
+
return held;
|
|
196
|
+
const stems = new Set(wordStems((0, factTerms_1.atoms)(text)));
|
|
197
|
+
if (text.length <= FIELD_STEMS_MAX_CHARS) {
|
|
198
|
+
if (fieldStems.size >= FIELD_STEMS_MAX)
|
|
199
|
+
fieldStems.delete(fieldStems.keys().next().value);
|
|
200
|
+
fieldStems.set(text, stems);
|
|
201
|
+
}
|
|
202
|
+
return stems;
|
|
203
|
+
}
|
|
204
|
+
/** The stems of the words a fact's saved statement was stored with (`statementTerms`: atoms joined by one space). */
|
|
205
|
+
function stemsOfStatementWords(statementWords) {
|
|
206
|
+
const text = statementWords ?? '';
|
|
207
|
+
const held = statementWordStems.get(text);
|
|
208
|
+
if (held)
|
|
209
|
+
return held;
|
|
210
|
+
const stems = new Set(wordStems(text ? text.split(' ') : []));
|
|
211
|
+
if (text.length <= FIELD_STEMS_MAX_CHARS) {
|
|
212
|
+
if (statementWordStems.size >= FIELD_STEMS_MAX)
|
|
213
|
+
statementWordStems.delete(statementWordStems.keys().next().value);
|
|
214
|
+
statementWordStems.set(text, stems);
|
|
215
|
+
}
|
|
216
|
+
return stems;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Score a fact's relevance to the query: matches in subject and predicate weight highest because they define what
|
|
220
|
+
* the fact is about; object matches are normalized by length (BM25 principle: long objects have incidental matches);
|
|
221
|
+
* distinct query term coverage scales the score; and facts matching the request's scope or naming the queried scope are preferred.
|
|
222
|
+
*/
|
|
223
|
+
function scoreFactRelevance(fact, query, requestScope) {
|
|
224
|
+
const queryTerms = factSearchTerms(query);
|
|
225
|
+
if (queryTerms.length === 0)
|
|
226
|
+
return 0;
|
|
227
|
+
return scoreFactTerms(fact, query, requestScope, queryTerms, queryTerms.map(cachedTermStems));
|
|
228
|
+
}
|
|
229
|
+
/** `scoreFactRelevance` for a query whose words, and their stems, the caller has already worked out. */
|
|
230
|
+
function scoreFactTerms(fact, query, requestScope, queryTerms, termStemList) {
|
|
231
|
+
const subjStems = stemsOfField(fact.subject);
|
|
232
|
+
const predStems = stemsOfField(fact.predicate);
|
|
233
|
+
const objStems = stemsOfField(fact.object ?? '');
|
|
234
|
+
const objectText = fact.object ?? '';
|
|
235
|
+
const objectWordTokens = objectText.trim() ? objectText.split(/\s+/).length : 0;
|
|
236
|
+
let subjectMatches = 0;
|
|
237
|
+
let predicateMatches = 0;
|
|
238
|
+
let objectMatches = 0;
|
|
239
|
+
let distinctMatched = 0;
|
|
240
|
+
for (const tStems of termStemList) {
|
|
241
|
+
const inSubj = tStems.some(s => subjStems.has(s));
|
|
242
|
+
const inPred = tStems.some(s => predStems.has(s));
|
|
243
|
+
const inObj = tStems.some(s => objStems.has(s));
|
|
244
|
+
if (inSubj)
|
|
245
|
+
subjectMatches++;
|
|
246
|
+
if (inPred)
|
|
247
|
+
predicateMatches++;
|
|
248
|
+
if (inObj)
|
|
249
|
+
objectMatches++;
|
|
250
|
+
if (inSubj || inPred || inObj)
|
|
251
|
+
distinctMatched++;
|
|
252
|
+
}
|
|
253
|
+
const subjectScore = subjectMatches * 4.0;
|
|
254
|
+
const predicateScore = predicateMatches * 4.0;
|
|
255
|
+
const bothBonus = subjectMatches > 0 && predicateMatches > 0 ? 3.0 : 0.0;
|
|
256
|
+
const lengthDamping = 1.0 + (objectWordTokens / 24.0);
|
|
257
|
+
const objectScore = (objectMatches * 2.0) / lengthDamping;
|
|
258
|
+
const coverage = distinctMatched / queryTerms.length;
|
|
259
|
+
const coverageMult = 1.0 + coverage;
|
|
260
|
+
let baseScore = (subjectScore + predicateScore + bothBonus + objectScore) * coverageMult;
|
|
261
|
+
if (requestScope && fact.scope) {
|
|
262
|
+
if ((0, scope_1.savedScopeMatches)(fact.scope, (0, scope_1.normalizeScope)(requestScope))) {
|
|
263
|
+
baseScore += 2.0;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
else if (!requestScope && fact.scope) {
|
|
267
|
+
const scopeSlug = fact.scope.includes(':') ? fact.scope.slice(fact.scope.indexOf(':') + 1) : fact.scope;
|
|
268
|
+
if (scopeSlug && query.toLowerCase().includes(scopeSlug.toLowerCase())) {
|
|
269
|
+
baseScore += 1.5;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return baseScore;
|
|
273
|
+
}
|
|
274
|
+
function rankCandidateFacts(facts, query, requestScope) {
|
|
275
|
+
if (!query.trim() || facts.length <= 1)
|
|
276
|
+
return facts;
|
|
277
|
+
const queryTerms = factSearchTerms(query);
|
|
278
|
+
const termStemList = queryTerms.map(cachedTermStems);
|
|
279
|
+
const scored = facts.map(fact => {
|
|
280
|
+
const subjStems = stemsOfField(fact.subject);
|
|
281
|
+
const predStems = stemsOfField(fact.predicate);
|
|
282
|
+
const objStems = stemsOfField(fact.object ?? '');
|
|
283
|
+
// The words of the statement the fact was saved with let a fact with a terse object reach the bar that the best fact
|
|
284
|
+
// sets from subject, predicate and object alone (bestDistinct below). A statement never raises that bar: it only
|
|
285
|
+
// admits its own fact. The order (scoreFactRelevance) does not change.
|
|
286
|
+
const stmtStems = stemsOfStatementWords(fact.statementWords);
|
|
287
|
+
let ownMatched = 0;
|
|
288
|
+
let distinctMatched = 0;
|
|
289
|
+
for (const tStems of termStemList) {
|
|
290
|
+
if (tStems.some(s => subjStems.has(s) || predStems.has(s) || objStems.has(s)))
|
|
291
|
+
ownMatched++;
|
|
292
|
+
if (tStems.some(s => subjStems.has(s) || predStems.has(s) || objStems.has(s) || stmtStems.has(s)))
|
|
293
|
+
distinctMatched++;
|
|
294
|
+
}
|
|
295
|
+
return {
|
|
296
|
+
fact,
|
|
297
|
+
score: queryTerms.length === 0 ? 0 : scoreFactTerms(fact, query, requestScope, queryTerms, termStemList),
|
|
298
|
+
ownMatched,
|
|
299
|
+
distinctMatched,
|
|
300
|
+
};
|
|
301
|
+
});
|
|
302
|
+
scored.sort((a, b) => {
|
|
303
|
+
if (Math.abs(b.score - a.score) > 1e-4) {
|
|
304
|
+
return b.score - a.score;
|
|
305
|
+
}
|
|
306
|
+
if (a.fact.validFrom !== b.fact.validFrom) {
|
|
307
|
+
return a.fact.validFrom < b.fact.validFrom ? 1 : -1;
|
|
308
|
+
}
|
|
309
|
+
return (b.fact.eventId ?? '').localeCompare(a.fact.eventId ?? '');
|
|
310
|
+
});
|
|
311
|
+
const bestDistinct = Math.max(...scored.map(s => s.ownMatched), 0);
|
|
312
|
+
if (bestDistinct <= 0)
|
|
313
|
+
return facts;
|
|
314
|
+
// A fact that shares fewer than half as many words as the best one is left out, counting the best one by its subject,
|
|
315
|
+
// predicate and object words only: when "Why did we consider moving Atlas to Fly.io?" finds the Fly.io fact by three
|
|
316
|
+
// words, a fact that only says "Atlas" is not what was asked.
|
|
317
|
+
return scored
|
|
318
|
+
.filter(item => item.distinctMatched * 2 >= bestDistinct)
|
|
319
|
+
.map(item => item.fact);
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* The maximum share of the facts lane's budget a single fact's object may take.
|
|
323
|
+
* Long fact objects (e.g. 2,000-char benchmark outputs from other lanes) are capped to this share so they do
|
|
324
|
+
* not crowd out subsequent answering facts in get_context.
|
|
325
|
+
*/
|
|
326
|
+
exports.MAX_FACT_OBJECT_LANE_SHARE = 0.35;
|
|
327
|
+
/**
|
|
328
|
+
* Appended to a fact object the lane cut, so a reader knows it is a prefix and where the rest is: the agent
|
|
329
|
+
* reads the whole fact with `read_memory` or `lookup_entity`.
|
|
330
|
+
*/
|
|
331
|
+
exports.TRUNCATION_MARKER = '… [truncated — read the rest with read_memory or lookup_entity]';
|
|
332
|
+
function capFactObject(object, maxTokens) {
|
|
333
|
+
if (replyTokens(object) <= maxTokens)
|
|
334
|
+
return object;
|
|
335
|
+
const budget = Math.max(1, maxTokens - replyTokens(exports.TRUNCATION_MARKER));
|
|
336
|
+
const words = object.split(/\s+/);
|
|
337
|
+
let low = 0;
|
|
338
|
+
let high = words.length;
|
|
339
|
+
let best = '';
|
|
340
|
+
while (low <= high) {
|
|
341
|
+
const mid = Math.floor((low + high) / 2);
|
|
342
|
+
const candidate = mid === 0 ? '' : words.slice(0, mid).join(' ') + '…';
|
|
343
|
+
if (replyTokens(candidate) <= budget) {
|
|
344
|
+
best = candidate;
|
|
345
|
+
low = mid + 1;
|
|
346
|
+
}
|
|
347
|
+
else {
|
|
348
|
+
high = mid - 1;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
return (best || (object.slice(0, Math.max(1, budget * 3)) + '…')) + exports.TRUNCATION_MARKER;
|
|
352
|
+
}
|
|
353
|
+
/** Whether a listed fact's object is `value`, or the prefix of it the lane cut (a cut object carries TRUNCATION_MARKER). */
|
|
354
|
+
const objectIs = (shown, value) => shown === value || (shown.includes('…') && value.startsWith(shown.split('…')[0]));
|
|
355
|
+
function keywordVariants(query) {
|
|
356
|
+
const compact = (0, factTerms_1.words)(query).filter(word => !exports.FTS_STOP_WORDS.has(word));
|
|
357
|
+
const variants = [query.trim()];
|
|
358
|
+
if (compact.length >= 2)
|
|
359
|
+
variants.push(compact.slice(0, 4).join(' '));
|
|
360
|
+
for (const word of compact.slice(0, 4))
|
|
361
|
+
variants.push(word);
|
|
362
|
+
return Array.from(new Set(variants.filter(Boolean))).slice(0, 6);
|
|
363
|
+
}
|
|
364
|
+
// get_context reads a fact as the user's own standing rule unless it says otherwise, so a fact another agent saved names
|
|
365
|
+
// it, and an import is marked untrusted. The reader's own facts stay bare: the reply is budgeted with these fields counted.
|
|
366
|
+
const toAgentFact = (fact, reader) => {
|
|
367
|
+
const { subject, predicate, object, validFrom, scope } = fact;
|
|
368
|
+
return { subject, predicate, object, validFrom, scope, ...(0, sourceName_1.authorMarker)(fact.author, reader) };
|
|
369
|
+
};
|
|
370
|
+
/**
|
|
371
|
+
* The words a predicate holding a standing rule or decision is saved under: what a later question about whether an
|
|
372
|
+
* action is allowed must be answered with, and what the action's own words never name. A predicate counts when it is
|
|
373
|
+
* one of these or is made of words one of which is (`ui_rule`, `approval_policy`, `deploy constraint`). The facts lane
|
|
374
|
+
* looks these up when the question asks for permission (`permissionQuestion`), because the question is about the action
|
|
375
|
+
* and shares no subject words with the rule that governs it. The words are read through the term index
|
|
376
|
+
* (`findCurrentFacts`), which is bounded by the query, not the table.
|
|
377
|
+
*
|
|
378
|
+
* A word outside this list is not a standing fact to the permission path, however it is worded (`note`, `status`): the
|
|
379
|
+
* list is the contract, and a governing fact saved under another word is found only by the question's own words.
|
|
380
|
+
*/
|
|
381
|
+
exports.GOVERNANCE_PREDICATES = [
|
|
382
|
+
'rule', 'rules', 'policy', 'policies', 'decision', 'decisions', 'guideline', 'guidelines',
|
|
383
|
+
'constraint', 'constraints', 'requirement', 'requirements', 'must', 'convention', 'conventions',
|
|
384
|
+
'principle', 'principles', 'guardrail', 'guardrails', 'permission', 'permissions', 'approval', 'approvals',
|
|
385
|
+
];
|
|
386
|
+
/** Whether a fact's predicate holds a standing rule or decision (`GOVERNANCE_PREDICATES`), whole or by one of its words. */
|
|
387
|
+
function isGovernancePredicate(predicate) {
|
|
388
|
+
const lower = predicate.toLowerCase();
|
|
389
|
+
if (exports.GOVERNANCE_PREDICATES.includes(lower))
|
|
390
|
+
return true;
|
|
391
|
+
return lower.split(/[^\p{L}\p{N}]+/u).some(word => word !== '' && exports.GOVERNANCE_PREDICATES.includes(word));
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* The most governance facts a permission question adds ahead of the facts its words found: enough for
|
|
395
|
+
* the rules that govern an action (there are usually a handful), never enough to crowd out the specific
|
|
396
|
+
* fact the question named.
|
|
397
|
+
*/
|
|
398
|
+
exports.PERMISSION_RULE_LIMIT = 3;
|
|
399
|
+
/**
|
|
400
|
+
* How many governance candidates are considered before the relevance ranking. Governance facts are a
|
|
401
|
+
* small share of a workspace, and the term index bounds the read (factTerms.ts), so this is generous
|
|
402
|
+
* while keeping the work bounded by the query. A workspace with more governance facts than this is not
|
|
403
|
+
* silently mis-ranked: the candidates are ranked by relevance to the question, and the ones sharing its
|
|
404
|
+
* words are the ones that matter.
|
|
405
|
+
*/
|
|
406
|
+
const PERMISSION_RULE_CANDIDATES = 1_000;
|
|
407
|
+
/**
|
|
408
|
+
* The phrases that ask whether an action is allowed or needs approval, as a person words them. The
|
|
409
|
+
* subject of such a question is the action ("Can I add a section to the dashboard?"), so it shares no
|
|
410
|
+
* words with the rule that governs it; the intent has to be read from the question, not matched to the
|
|
411
|
+
* rule. Matched on word boundaries over a lower-cased, punctuation-stripped query, contractions spelled out.
|
|
412
|
+
*
|
|
413
|
+
* These ask for permission wherever they are: a modal addressed to the speaker, or a need for approval.
|
|
414
|
+
*/
|
|
415
|
+
const PERMISSION_PHRASES = [
|
|
416
|
+
'can i', 'can we', 'could i', 'could we', 'may i', 'may we', 'should i', 'should we',
|
|
417
|
+
'am i allowed', 'are we allowed', 'is it allowed', 'is this allowed', 'is that allowed', 'is that still allowed',
|
|
418
|
+
'need permission', 'needs permission', 'need approval', 'needs approval', 'need an approval',
|
|
419
|
+
'require approval', 'requires approval', 'am i permitted', 'are we permitted', 'is it permitted',
|
|
420
|
+
'need to ask', 'needs to ask', 'have to ask', 'has to ask', 'ask before', 'ask first', 'ask permission',
|
|
421
|
+
'need to check with', 'need to confirm with', 'have to check with',
|
|
422
|
+
'do i have to', 'do we have to',
|
|
423
|
+
'is it ok to', 'is it okay to', 'is it fine to', 'is it acceptable to', 'is that ok', 'is that okay',
|
|
424
|
+
'would it be okay to', 'would it be ok to', 'would it be fine to', 'would it be acceptable to',
|
|
425
|
+
];
|
|
426
|
+
/**
|
|
427
|
+
* Words that ask for permission only in a question about the present: "Which regions are allowed?" asks, "The
|
|
428
|
+
* migration is allowed to run after review." states, and "Where did we check with the vendor?" recalls (delta review
|
|
429
|
+
* P3, 2026-10-09).
|
|
430
|
+
*/
|
|
431
|
+
const PERMISSION_WORDS = [
|
|
432
|
+
'allowed', 'permitted', 'permission to', 'permission for',
|
|
433
|
+
'check with', 'confirm with', 'clear it with', 'clear this with', 'okay with', 'ok with',
|
|
434
|
+
];
|
|
435
|
+
/** Openings that make a sentence a question without its question mark. */
|
|
436
|
+
const QUESTION_OPENINGS = /^(?:is|are|am|do|does|can|could|may|might|should|would|will|must|what|which|who|whom|whose|where|when|why|how)\b/;
|
|
437
|
+
/** Openings of a question about what was done, not what may be: "Where did we", "When was it". */
|
|
438
|
+
const RECOLLECTION = /^(?:what|which|who|where|when|why|how)\s+(?:did|was|were|had)\b/;
|
|
439
|
+
/** Whether `query` asks whether an action is allowed or needs approval. */
|
|
440
|
+
function permissionQuestion(query) {
|
|
441
|
+
const spelled = query.toLowerCase()
|
|
442
|
+
.replace(/[’']/g, "'")
|
|
443
|
+
.replace(/\bcan't\b/g, 'can not').replace(/\bcannot\b/g, 'can not').replace(/\bwon't\b/g, 'will not')
|
|
444
|
+
.replace(/\b(\p{L}+)n't\b/gu, '$1 not');
|
|
445
|
+
const words = spelled.replace(/[^\p{L}\p{N}\s]+/gu, ' ').replace(/\s+/g, ' ').trim();
|
|
446
|
+
if (!words)
|
|
447
|
+
return false;
|
|
448
|
+
// "can not i" / "could not we" ask what "can i" asks: the negation is the asker's doubt, not the action.
|
|
449
|
+
const padded = ` ${words.replace(/\b(can|could|may|should|would|do) not (i|we)\b/g, '$1 $2')} `;
|
|
450
|
+
if (PERMISSION_PHRASES.some(phrase => padded.includes(` ${phrase} `)))
|
|
451
|
+
return true;
|
|
452
|
+
const asks = query.trim().endsWith('?') || QUESTION_OPENINGS.test(words);
|
|
453
|
+
return asks && !RECOLLECTION.test(words) && PERMISSION_WORDS.some(phrase => padded.includes(` ${phrase} `));
|
|
454
|
+
}
|
|
455
|
+
function governanceCandidates(db, scope) {
|
|
456
|
+
return (0, factTerms_1.findCurrentFacts)(db, {
|
|
457
|
+
terms: exports.GOVERNANCE_PREDICATES,
|
|
458
|
+
scope,
|
|
459
|
+
match: 'any',
|
|
460
|
+
limit: PERMISSION_RULE_CANDIDATES,
|
|
461
|
+
}).filter(fact => isGovernancePredicate(fact.predicate)); // a note that says "decision" is not one (delta review P3)
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* The standing facts' closeness in meaning to a permission question (`PermissionMeaning`). `queryVector` is the one
|
|
465
|
+
* the read's search by meaning embedded, when it ran; otherwise the question is embedded here, through the same
|
|
466
|
+
* provider.
|
|
467
|
+
*
|
|
468
|
+
* It runs only for an embedding model with a measured bar (CALIBRATED_BARS: today the Vertex model production runs),
|
|
469
|
+
* and with that bar. An explicit RENOR_CONTEXT_RELEVANCE_MIN tunes the relevance signal; it is not a calibration for
|
|
470
|
+
* this lookup, so any other embedder skips it and the standing facts are found by words alone (Batch 7 follow-up c).
|
|
471
|
+
*/
|
|
472
|
+
async function permissionMeaning(db, engine, query, scope, queryVector) {
|
|
473
|
+
const provider = engine.provider;
|
|
474
|
+
const bar = engine.modelKey && engine.modelKey in contextRelevance_1.CALIBRATED_BARS ? contextRelevance_1.CALIBRATED_BARS[engine.modelKey] : null;
|
|
475
|
+
if (!provider || bar === null || !permissionQuestion(query))
|
|
476
|
+
return undefined;
|
|
477
|
+
try {
|
|
478
|
+
const candidates = governanceCandidates(db, scope === undefined ? undefined : (0, scope_1.normalizeScope)(scope));
|
|
479
|
+
if (candidates.length === 0)
|
|
480
|
+
return undefined;
|
|
481
|
+
const vector = queryVector ?? (provider.embedQuery ? await provider.embedQuery(query) : (await provider.embed([query]))[0]);
|
|
482
|
+
if (!vector)
|
|
483
|
+
return undefined;
|
|
484
|
+
const similarity = await (0, contextRelevance_1.similaritiesTo)(provider, candidates.map(f => (0, contextRelevance_1.renderFactForRelevance)(f.predicate, f.object ?? '')), vector);
|
|
485
|
+
return similarity ? { similarity, bar } : undefined;
|
|
486
|
+
}
|
|
487
|
+
catch {
|
|
488
|
+
return undefined;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
/**
|
|
492
|
+
* The standing facts a permission question should be answered with, at most `PERMISSION_RULE_LIMIT`:
|
|
493
|
+
*
|
|
494
|
+
* 1. the ones close in meaning to the question (at or over the calibrated bar, `permissionMeaning`), closest first:
|
|
495
|
+
* a rule worded without any of the action's words ("Ship nothing until Rowan signs off" for "Can I deploy?");
|
|
496
|
+
* 2. then the ones sharing a word with the question, ranked the way the word-found facts are (`scoreFactRelevance`:
|
|
497
|
+
* subject and predicate overlap, a length-damped object, the named-scope bonus), then newest. On a scoped read the
|
|
498
|
+
* scope's own name counts as one of the question's words: an agent scoped to project:renor asks "Can I add a
|
|
499
|
+
* section to the dashboard?", and the rule about Renor is about its question (delta review P2, 2026-10-09);
|
|
500
|
+
* 3. when none of them share a word, the newest.
|
|
501
|
+
*
|
|
502
|
+
* Without a provider or a bar, or when the backend fails, the list is 2 and 3, as it was.
|
|
503
|
+
*/
|
|
504
|
+
function permissionFacts(db, query, scope, meaning) {
|
|
505
|
+
const candidates = governanceCandidates(db, scope);
|
|
506
|
+
if (candidates.length === 0)
|
|
507
|
+
return [];
|
|
508
|
+
const slug = scope?.includes(':') ? scope.slice(scope.indexOf(':') + 1) : scope;
|
|
509
|
+
const asked = slug && slug !== 'global' && !query.toLowerCase().includes(slug.toLowerCase()) ? `${query} ${slug}` : query;
|
|
510
|
+
const newestFirst = (a, b) => {
|
|
511
|
+
if (a.validFrom !== b.validFrom)
|
|
512
|
+
return a.validFrom < b.validFrom ? 1 : -1;
|
|
513
|
+
return (b.eventId ?? '').localeCompare(a.eventId ?? '');
|
|
514
|
+
};
|
|
515
|
+
const close = meaning
|
|
516
|
+
? candidates
|
|
517
|
+
.map(fact => ({ fact, similarity: meaning.similarity.get((0, contextRelevance_1.renderFactForRelevance)(fact.predicate, fact.object ?? '')) ?? -Infinity }))
|
|
518
|
+
.filter(item => Math.round(item.similarity * 1000) / 1000 >= meaning.bar)
|
|
519
|
+
.sort((a, b) => (Math.abs(b.similarity - a.similarity) > 1e-6 ? b.similarity - a.similarity : newestFirst(a.fact, b.fact)))
|
|
520
|
+
.map(item => item.fact)
|
|
521
|
+
: [];
|
|
522
|
+
const scored = candidates
|
|
523
|
+
.map(fact => ({ fact, score: scoreFactRelevance(fact, asked, scope) }))
|
|
524
|
+
.sort((a, b) => (Math.abs(b.score - a.score) > 1e-4 ? b.score - a.score : newestFirst(a.fact, b.fact)));
|
|
525
|
+
const related = scored.filter(item => item.score > 0);
|
|
526
|
+
const byWords = (related.length > 0 ? related : scored).map(item => item.fact);
|
|
527
|
+
const out = [];
|
|
528
|
+
for (const fact of [...close, ...byWords]) {
|
|
529
|
+
if (out.length === exports.PERMISSION_RULE_LIMIT)
|
|
530
|
+
break;
|
|
531
|
+
if (!out.includes(fact))
|
|
532
|
+
out.push(fact);
|
|
533
|
+
}
|
|
534
|
+
return out;
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
537
|
+
* The standing facts a permission question put ahead of the facts its words found, for this read: get_context caps a
|
|
538
|
+
* long one that leads the page (`capFact`). Marked on the fact itself, so a word-found fact with a standing predicate
|
|
539
|
+
* is not taken for one, and a fact the permission path put first is, whatever its predicate (delta review P3).
|
|
540
|
+
*/
|
|
541
|
+
const STANDING = new WeakSet();
|
|
542
|
+
/**
|
|
543
|
+
* Current lifecycle facts for a query: the facts found by its words (`factQueryTerms`, and two neighbouring words
|
|
544
|
+
* joined, so "time zone" finds a `timezone` key), ranked by query relevance and scope preference, then newest. A fact
|
|
545
|
+
* that scores less than half as much as the best one is left out. With no query, the newest facts.
|
|
546
|
+
*
|
|
547
|
+
* A question about whether an action is allowed or needs approval is also about the standing rule or
|
|
548
|
+
* decision that governs it, which its words do not name. Those facts are looked up through the term
|
|
549
|
+
* index (`permissionFacts`) and put first, ahead of the facts the words found: the rule is the answer,
|
|
550
|
+
* and the long other-lane facts that share the action's words used to fill the candidate window before
|
|
551
|
+
* the rule was even ranked (2026-10-09). They are ranked by meaning where the embedding path can say, then by
|
|
552
|
+
* relevance to the question, not by recency, deduped against the word-found facts, and capped.
|
|
553
|
+
*/
|
|
554
|
+
function currentLifecycleFacts(db, query = '', scope, limit = 12) {
|
|
555
|
+
return currentFactMatches(db, query, scope, limit).map(fact => toAgentFact(fact));
|
|
556
|
+
}
|
|
557
|
+
/** `currentLifecycleFacts` with each fact's ledger event, which get_context reads to list an event once. */
|
|
558
|
+
function currentFactMatches(db, query, scope, limit, meaning) {
|
|
559
|
+
const scopeNorm = scope === undefined ? undefined : (0, scope_1.normalizeScope)(scope);
|
|
560
|
+
if (!query.trim())
|
|
561
|
+
return (0, factTerms_1.recentCurrentFacts)(db, scopeNorm, limit);
|
|
562
|
+
const found = (0, factTerms_1.findCurrentFacts)(db, {
|
|
563
|
+
terms: [...factQueryTerms(query), ...joinedWords(query)],
|
|
564
|
+
scope: scopeNorm,
|
|
565
|
+
match: 'any',
|
|
566
|
+
limit: Math.max(limit * 3, 36),
|
|
567
|
+
});
|
|
568
|
+
const ranked = rankCandidateFacts(found, query, scopeNorm);
|
|
569
|
+
if (!permissionQuestion(query))
|
|
570
|
+
return ranked.slice(0, limit);
|
|
571
|
+
const standing = permissionFacts(db, query, scopeNorm, meaning);
|
|
572
|
+
if (standing.length === 0)
|
|
573
|
+
return ranked.slice(0, limit);
|
|
574
|
+
const key = (fact) => `${fact.scope ?? 'global'}\u0000${fact.subject}\u0000${fact.predicate}`;
|
|
575
|
+
const seen = new Set(ranked.map(key));
|
|
576
|
+
const prepended = standing.filter(fact => !seen.has(key(fact)));
|
|
577
|
+
for (const fact of prepended)
|
|
578
|
+
STANDING.add(fact);
|
|
579
|
+
return [...prepended, ...ranked].slice(0, limit);
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* Whether a name is on file for a read: a current fact or a note (its text, path or front matter) of the scope read has
|
|
583
|
+
* its words. Read through the indexes the facts lane and search_text read, so a scoped read never consults another scope.
|
|
584
|
+
*/
|
|
585
|
+
function nameOnFile(db, memoryDir, { scope, includeHistory = false, ledgerRelPath } = {}) {
|
|
586
|
+
return name => {
|
|
587
|
+
const terms = (0, factTerms_1.atoms)(name);
|
|
588
|
+
if (terms.length === 0)
|
|
589
|
+
return true; // nothing to look up is no evidence
|
|
590
|
+
if ((0, factTerms_1.findCurrentFacts)(db, { terms, scope, match: 'all', limit: 1 }).length > 0)
|
|
591
|
+
return true;
|
|
592
|
+
const keyword = (0, engine_1.engineSearchText)(db, memoryDir, terms.join(' '), 1, scope, includeHistory, ledgerRelPath);
|
|
593
|
+
return !keyword.ok || keyword.results.length > 0;
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
/** Whether `hit` is the semantic hit the probe scored best, as it is listed. */
|
|
597
|
+
function isTopHit(top, hit) {
|
|
598
|
+
return top !== undefined && hit.notePath === top.notePath && (hit.headingPath ?? '') === (top.headingPath || '')
|
|
599
|
+
&& hit.text.trim() === top.text.trim();
|
|
600
|
+
}
|
|
601
|
+
/**
|
|
602
|
+
* The places `top_k` leaves for facts on a page of `indexHits`: what the index's own hits do not fill, the leading
|
|
603
|
+
* evidence summary aside (that one is outside top_k by the tool's own promise). Zero on a page top_k has already
|
|
604
|
+
* filled, which is the only case in which a caller cannot list a fact it wants to.
|
|
605
|
+
*/
|
|
606
|
+
function factLaneRoom(indexHits, limit) {
|
|
607
|
+
const lead = (0, digests_1.leadingSummaryHits)(indexHits);
|
|
608
|
+
return Math.max(0, limit - (indexHits.length - lead));
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* search_memory's relevance (RENOR_CONTEXT_RELEVANCE, where armed): get_context's verdict over the same read. A
|
|
612
|
+
* search adds no hit for the verdict, so "relevant" stands only when the hits hold the match it reports: the closest
|
|
613
|
+
* returned note chunk, or the closest fact (as a fact hit or its ledger event). A relevant verdict whose fact the
|
|
614
|
+
* hits do not hold is returned with that fact in `listFact` for the caller to list — the same rule get_context applies
|
|
615
|
+
* to the fact its own verdict was judged on (`missed`), and for the same reason: the fact lane exists because the
|
|
616
|
+
* ledger chunk behind a fact is boilerplate that does not rank (contextRelevance.ts), so on a workspace whose notes
|
|
617
|
+
* out-rank that chunk — and whose full page leaves the facts lane no room — the fact was not going to be listed, and
|
|
618
|
+
* withholding the verdict there left the agent with no way to tell "the answer is on file" from "cannot judge".
|
|
619
|
+
* `limit` is the page's top_k, which decides that case: with room the caller lists the fact, and without it the
|
|
620
|
+
* verdict names the fact itself (`relevance.fact`) rather than the page passing top_k. The chunk lane needs no such
|
|
621
|
+
* rule: the probe's best chunk is the hit it was taken from, so it is always held.
|
|
622
|
+
*
|
|
623
|
+
* `hits` are the index's own hits, before the facts lane adds any: a fact the hits already carry is not named twice.
|
|
624
|
+
*/
|
|
625
|
+
function searchRelevance(db, engine, memoryDir, read, probe, hits, limit) {
|
|
626
|
+
if (!(0, contextRelevance_1.contextRelevanceEnabled)(read.env))
|
|
627
|
+
return {};
|
|
628
|
+
const scope = read.scope === undefined ? undefined : (0, scope_1.normalizeScope)(read.scope);
|
|
629
|
+
const judged = (0, contextRelevance_1.assessQueryRelevance)({
|
|
630
|
+
db, engine, scope, probe, env: read.env,
|
|
631
|
+
query: read.query, onFile: nameOnFile(db, memoryDir, { scope, includeHistory: read.includeHistory, ledgerRelPath: read.ledgerRelPath }),
|
|
632
|
+
});
|
|
633
|
+
if (!judged)
|
|
634
|
+
return {};
|
|
635
|
+
if (!judged.relevance.relevant)
|
|
636
|
+
return { relevance: judged.relevance };
|
|
637
|
+
const held = judged.matchedOn === 'chunk'
|
|
638
|
+
? hits.some(hit => isTopHit(probe.topHit, hit))
|
|
639
|
+
: (() => {
|
|
640
|
+
const event = judged.closestFact?.eventId?.toLowerCase();
|
|
641
|
+
return event !== undefined && hits.some(hit => (0, ledgerLiveness_1.ledgerEventIds)(hit).includes(event));
|
|
642
|
+
})();
|
|
643
|
+
if (held)
|
|
644
|
+
return { relevance: judged.relevance };
|
|
645
|
+
if (judged.matchedOn !== 'fact' || !judged.closestFact)
|
|
646
|
+
return {};
|
|
647
|
+
// A page top_k has already filled takes no more hits, so the fact cannot be listed; naming it is what keeps the
|
|
648
|
+
// verdict honest without breaking the tool's own bound (factsLane `also`).
|
|
649
|
+
if (limit !== undefined && factLaneRoom(hits, limit) === 0) {
|
|
650
|
+
const { subject, predicate, object, scope: factScope } = judged.closestFact;
|
|
651
|
+
return { relevance: { ...judged.relevance, fact: { subject, predicate, object, scope: (0, scope_1.normalizeScope)(factScope) } } };
|
|
652
|
+
}
|
|
653
|
+
return { relevance: judged.relevance, listFact: judged.closestFact };
|
|
654
|
+
}
|
|
655
|
+
/** A note the import tool wrote (importing/hosted.ts: `Imported/<label>/…`), never one of the engine's own notes. */
|
|
656
|
+
const IMPORTED_PATH = /^Imported\//;
|
|
657
|
+
function normalizeMemory(hit, fallbackReason, reader) {
|
|
658
|
+
if (!hit || typeof hit.text !== 'string' || !hit.text.trim())
|
|
659
|
+
return null;
|
|
660
|
+
const reason = typeof hit.reason === 'string' ? hit.reason : fallbackReason;
|
|
661
|
+
return {
|
|
662
|
+
notePath: typeof hit.notePath === 'string' ? hit.notePath : '',
|
|
663
|
+
...(typeof hit.headingPath === 'string' && hit.headingPath ? { headingPath: hit.headingPath } : {}),
|
|
664
|
+
text: hit.text,
|
|
665
|
+
...(typeof hit.score === 'number' ? { score: hit.score } : {}),
|
|
666
|
+
reason,
|
|
667
|
+
...(typeof hit.session_date === 'string' ? { session_date: hit.session_date } : {}),
|
|
668
|
+
// A fact the retrieval named its author of (a fact an identity question reads) is marked as get_context's facts are.
|
|
669
|
+
// A memory that is a ledger event is marked in buildAgentContext, which reads its event's author.
|
|
670
|
+
...(hit.author !== undefined ? (0, sourceName_1.authorMarker)(hit.author, reader) : {}),
|
|
671
|
+
// A statement the session expansion added sits in the same memories[] list as matched evidence, so the item says
|
|
672
|
+
// in a field a reader reads that the query did not match it (adv hardening, report renor-adv-inj5-1011 proposal 3):
|
|
673
|
+
// an explicit boolean, in the marker style the hive tools use for untrusted agent text (memory/whatsNew.ts). The
|
|
674
|
+
// search lane is untouched, so a matched memory — and every search_memory hit — keeps the fields it had, byte for
|
|
675
|
+
// byte. The marker's tokens are budget tokens like any other item's (applyBudget `cost`).
|
|
676
|
+
...(reason.split('+').includes(orchestrator_1.SESSION_EXPAND_REASON) ? { matched: false } : {}),
|
|
677
|
+
...(IMPORTED_PATH.test(String(hit.notePath ?? '')) ? { origin: 'import' } : {}),
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
exports.CONTEXT_LEAN_ITEMS_FLAG = 'RENOR_CONTEXT_LEAN_ITEMS';
|
|
681
|
+
/** get_context's lean memory items (RENOR_CONTEXT_LEAN_ITEMS): off by default, and the reply is then what it always was. */
|
|
682
|
+
function contextLeanItemsEnabled(env = process.env) {
|
|
683
|
+
const raw = (env[exports.CONTEXT_LEAN_ITEMS_FLAG] ?? '').trim().toLowerCase();
|
|
684
|
+
return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* A memory as the lean get_context lists it: its text as stored, the day it was written, and every marker it carries
|
|
688
|
+
* (replaced, matched, origin, source, untrusted). The note path, heading, score and reason go: they cost about twice
|
|
689
|
+
* the memory's own text (an item was about 107 tokens for 32 of text, docs/evals/results/agent-workflow-20261020), and
|
|
690
|
+
* the reply is already in the order they decided. A daily entry's text keeps its own time and writer.
|
|
691
|
+
*/
|
|
692
|
+
function leanMemory(memory) {
|
|
693
|
+
const { notePath: _notePath, headingPath: _headingPath, text, score: _score, reason: _reason, session_date, ...markers } = memory;
|
|
694
|
+
const day = session_date === undefined ? undefined : /^\d{4}-\d{2}-\d{2}/.exec(session_date)?.[0] ?? session_date;
|
|
695
|
+
return { text, ...(day === undefined ? {} : { session_date: day }), ...markers };
|
|
696
|
+
}
|
|
697
|
+
/** The reasons of a memory the question's own words found (keyword search), as opposed to one found by meaning alone. */
|
|
698
|
+
const WORD_MATCH_REASONS = ['fts', 'note-identity'];
|
|
699
|
+
function dedupeMemories(memories) {
|
|
700
|
+
const at = new Map();
|
|
701
|
+
const out = [];
|
|
702
|
+
for (const memory of memories) {
|
|
703
|
+
const key = `${memory.notePath}\0${memory.headingPath ?? ''}\0${memory.text.trim()}`;
|
|
704
|
+
const kept = at.get(key);
|
|
705
|
+
if (kept === undefined) {
|
|
706
|
+
at.set(key, out.length);
|
|
707
|
+
out.push(memory);
|
|
708
|
+
}
|
|
709
|
+
else if (WORD_MATCH_REASONS.includes(memory.reason) && !out[kept].reason.split('+').includes(memory.reason)) {
|
|
710
|
+
// The question's words found a memory that is already listed (by meaning, usually): the one copy keeps saying so,
|
|
711
|
+
// where it used to be dropped without a trace, and a caller cannot otherwise tell "close in meaning" from "contains
|
|
712
|
+
// what you asked for".
|
|
713
|
+
out[kept] = { ...out[kept], reason: `${out[kept].reason}+${memory.reason}` };
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
return out;
|
|
717
|
+
}
|
|
718
|
+
/**
|
|
719
|
+
* The most of the budget facts take while memories are waiting; the memories get the rest, and the room they leave
|
|
720
|
+
* goes back to the facts. Facts used to be added until the budget was spent, so a dozen facts that each shared one
|
|
721
|
+
* word with the question cut the note that answered it.
|
|
722
|
+
*/
|
|
723
|
+
exports.FACTS_BUDGET_SHARE = 0.4;
|
|
724
|
+
/**
|
|
725
|
+
* The most of the budget the newest match above the relevance bar and the rest of its own session may take, before the
|
|
726
|
+
* others are taken by score (applyBudget): the match itself, its other matches above the bar, and the statements the
|
|
727
|
+
* session's retrieval expansion added. A session that said something newer about the question said it in more than one
|
|
728
|
+
* chunk, and the one that holds the answer is often not its closest: with the reservation alone, a closer chunk of an older
|
|
729
|
+
* session took the room and the later statement was cut. A quarter of the budget (the first value, PR 508) left a
|
|
730
|
+
* budget of 800 tokens room for one big chunk of the session and no statement beside it; the practice questions were best
|
|
731
|
+
* served at 0.6, and worse at 0.5 and 0.4 (docs/evals/results/NEWEST-FACT-POOL-20261011.md).
|
|
732
|
+
*/
|
|
733
|
+
exports.NEWEST_SESSION_SHARE = 0.6;
|
|
734
|
+
/**
|
|
735
|
+
* The most pieces of the newest session the reservation takes: the match, its other matches and its statements together.
|
|
736
|
+
* They are taken by relevance per token, so the share above would otherwise be spent on as many small pieces as fit, and
|
|
737
|
+
* four is what the 1,200-token budget holds beside the older evidence without losing a question the cut by score kept.
|
|
738
|
+
*/
|
|
739
|
+
exports.NEWEST_SESSION_PIECES = 4;
|
|
740
|
+
/**
|
|
741
|
+
* The fewest of the question's content words (`statementRelevance`) a statement of the newest session must hold to be
|
|
742
|
+
* a candidate for its reservation. The reservation ranks its candidates by relevance squared per token, so a statement with
|
|
743
|
+
* one shared word takes room only when nothing better fits; one with none is unmatched text and never takes it.
|
|
744
|
+
*/
|
|
745
|
+
exports.NEWEST_SESSION_STATEMENT_FLOOR = 1;
|
|
746
|
+
/**
|
|
747
|
+
* How much more a session statement weighs against a chunk of the same relevance. A statement is a short piece of what
|
|
748
|
+
* the user said (the expansion cuts them at 100 tokens); a chunk is 250 to 500 tokens of the answer around it.
|
|
749
|
+
*/
|
|
750
|
+
exports.NEWEST_SESSION_STATEMENT_WEIGHT = 2;
|
|
751
|
+
/**
|
|
752
|
+
* The most statements that share a single word of the question the reservation takes. One shared word is a coincidence, and the
|
|
753
|
+
* deciding statement can be one (a turn that says "I now have 5 of them" shares "bikes"), but a session of many short turns
|
|
754
|
+
* about the same subject has several, and each weighs double and costs half a chunk: four of them took the room of the scored
|
|
755
|
+
* chunk that held the answer (a held-out question about how many bikes a person owns, after the statement cap went to 120
|
|
756
|
+
* tokens). A statement that shares two words, and every match retrieval scored, are not counted.
|
|
757
|
+
*/
|
|
758
|
+
exports.NEWEST_SESSION_WEAK_STATEMENTS = 1;
|
|
759
|
+
/** The fewest of the question's content words the newest match above the bar must share with it (`newestMatchAbove`). */
|
|
760
|
+
exports.NEWEST_ANCHOR_RELEVANCE_FLOOR = 1;
|
|
761
|
+
/** Tokens of a reply as the engine counts them: o200k, the chunker's tokenizer, special-token syntax read as text.
|
|
762
|
+
* The reply is escaped after fitting (jsonResult), so the budget counts what is actually sent: a memory
|
|
763
|
+
* rich in hidden characters must not push the reply past max_tokens once they become \uXXXX escapes. */
|
|
764
|
+
function replyTokens(text) {
|
|
765
|
+
return (0, tokenizer_1.encodeTokens)((0, sanitize_1.escapeInvisibleControls)(text));
|
|
766
|
+
}
|
|
767
|
+
/**
|
|
768
|
+
* The order the matches are taken into a token budget in, as positions in `memories`: first the summary tables that lead
|
|
769
|
+
* the list, when there are any (the evidence and digest tables, `leadingSummaryHits`: they sit outside any order), then the
|
|
770
|
+
* matches retrieval scored, best score first, then the matches the words of the request found, as they are listed. A match
|
|
771
|
+
* retrieval scored is one whose score is the one that picked the window (cosine plus its boosts), so those compare; keyword
|
|
772
|
+
* matches are scored on other scales (WORD_MATCH_REASONS) and are not ranked with them. Equal scores keep their place in the
|
|
773
|
+
* list, which makes the order the same on every call, and under "score" (a list already in score order) the order is the
|
|
774
|
+
* list's own. What the session expansion added is no match and is not in it.
|
|
775
|
+
*
|
|
776
|
+
* The budget keeps the longest run of this order that fits (applyBudget), except that a top match too big for the whole budget is
|
|
777
|
+
* passed over. The list is laid out newest first, and a budget that kept the longest run of the layout cut the oldest matches
|
|
778
|
+
* first however close they were: retrieval ablations, 2026-10-06 (docs/evals/results/CUT-BY-SCORE-20261007.md, which also reads what the cut by score trades away).
|
|
779
|
+
*/
|
|
780
|
+
function matchBudgetOrder(memories) {
|
|
781
|
+
const leading = (0, digests_1.leadingSummaryHits)(memories);
|
|
782
|
+
const lead = [];
|
|
783
|
+
const scored = [];
|
|
784
|
+
const found = [];
|
|
785
|
+
memories.forEach((memory, i) => {
|
|
786
|
+
if (memory.reason === orchestrator_1.SESSION_EXPAND_REASON)
|
|
787
|
+
return;
|
|
788
|
+
if (i < leading)
|
|
789
|
+
lead.push(i);
|
|
790
|
+
else if (memory.reason.split('+').every(reason => WORD_MATCH_REASONS.includes(reason)))
|
|
791
|
+
found.push(i);
|
|
792
|
+
else
|
|
793
|
+
scored.push(i);
|
|
794
|
+
});
|
|
795
|
+
// A score that is not a number ranks last among the scored matches, never first: it cannot displace a match.
|
|
796
|
+
const score = (i) => (Number.isFinite(memories[i].score) ? memories[i].score : Number.NEGATIVE_INFINITY);
|
|
797
|
+
scored.sort((a, b) => score(b) - score(a) || a - b);
|
|
798
|
+
return [...lead, ...scored, ...found];
|
|
799
|
+
}
|
|
800
|
+
/**
|
|
801
|
+
* The newest match above the relevance bar, as a position in `memories` (-1 when there is none): the match a cut by score
|
|
802
|
+
* must not take the place of. A cut by score keeps the closest matches, and when two statements are about one thing it
|
|
803
|
+
* cannot tell which is current; the newest one that is itself relevant (its raw cosine, `similarities`, is at least `bar`,
|
|
804
|
+
* the bar the relevance signal judges by) is the one the reader needs last. A match that is not a semantic hit of this
|
|
805
|
+
* read (a keyword match, a statement the expansion added) has no cosine and is never it, and neither is one with no date.
|
|
806
|
+
* The newer date wins; of equal dates the earlier in the list, which is the better score.
|
|
807
|
+
*
|
|
808
|
+
* With the question (`query`), a match must also share NEWEST_ANCHOR_RELEVANCE_FLOOR of its content words to be it: the
|
|
809
|
+
* cosine is a measure of meaning, and a newer session about something else (a fashion chat that answers a question about
|
|
810
|
+
* a grocery list) can sit above the bar and would take the reservation, and the session's room, from the session
|
|
811
|
+
* that answers. A question with no word to share (only numbers and single letters, `statementRelevance` counts neither) leaves the bar alone to decide.
|
|
812
|
+
*/
|
|
813
|
+
/** First similarity for each hit, keyed the way `isTopHit` compares a memory to a hit. */
|
|
814
|
+
function similarityByMemory(similarities) {
|
|
815
|
+
const byNote = new Map();
|
|
816
|
+
for (const hit of similarities) {
|
|
817
|
+
const heading = hit.headingPath || '';
|
|
818
|
+
const text = hit.text.trim();
|
|
819
|
+
let byHeading = byNote.get(hit.notePath);
|
|
820
|
+
if (!byHeading)
|
|
821
|
+
byNote.set(hit.notePath, byHeading = new Map());
|
|
822
|
+
let byText = byHeading.get(heading);
|
|
823
|
+
if (!byText)
|
|
824
|
+
byHeading.set(heading, byText = new Map());
|
|
825
|
+
if (!byText.has(text))
|
|
826
|
+
byText.set(text, hit.similarity);
|
|
827
|
+
}
|
|
828
|
+
return memory => byNote.get(memory.notePath)?.get(memory.headingPath ?? '')?.get(memory.text.trim());
|
|
829
|
+
}
|
|
830
|
+
function newestMatchAbove(memories, similarities, bar, query) {
|
|
831
|
+
let newest = -1;
|
|
832
|
+
let newestAt = Number.NEGATIVE_INFINITY;
|
|
833
|
+
const asked = query === undefined ? 0 : questionTerms(query).length;
|
|
834
|
+
// The cosine rules out almost every memory. Stemming a statement the cosine has already rejected cannot change the pick.
|
|
835
|
+
const heardAt = similarityByMemory(similarities);
|
|
836
|
+
memories.forEach((memory, i) => {
|
|
837
|
+
if (memory.reason === orchestrator_1.SESSION_EXPAND_REASON || !memory.session_date)
|
|
838
|
+
return;
|
|
839
|
+
const heard = heardAt(memory);
|
|
840
|
+
if (heard === undefined || !(heard >= bar))
|
|
841
|
+
return;
|
|
842
|
+
if (asked > 0 && statementRelevance(memory.text, query) < exports.NEWEST_ANCHOR_RELEVANCE_FLOOR)
|
|
843
|
+
return;
|
|
844
|
+
const at = (0, orchestrator_1.sessionDateMs)(memory.session_date);
|
|
845
|
+
if (at > newestAt) {
|
|
846
|
+
newest = i;
|
|
847
|
+
newestAt = at;
|
|
848
|
+
}
|
|
849
|
+
});
|
|
850
|
+
return newest;
|
|
851
|
+
}
|
|
852
|
+
/** The session a match belongs to: its note and its date. A chat session is one note with one date; a ledger or a daily log holds one session per event or day. */
|
|
853
|
+
const sessionOf = (memory) => `${memory.notePath}\0${memory.session_date ?? ''}`;
|
|
854
|
+
/** The content words of a question: the ones a statement can share with it (`statementRelevance`). */
|
|
855
|
+
function questionTerms(query) {
|
|
856
|
+
return factQueryTerms(query).filter(term => term.length > 1 && !/^\p{Nd}+$/u.test(term));
|
|
857
|
+
}
|
|
858
|
+
/**
|
|
859
|
+
* How many distinct content words of the question a text holds, as stems: how much a statement says about the
|
|
860
|
+
* question that brought its session. The session expansion adds a statement without scoring it, and the budget that
|
|
861
|
+
* keeps the newest session's statements can rarely keep all of them: the one that answers the question is usually
|
|
862
|
+
* not the first the session said (a header, a turn about something else), so they are taken most relevant first.
|
|
863
|
+
* A word of one letter or one made of digits alone never says what a statement is about — "I" is in every statement
|
|
864
|
+
* of a chat, and "10 pounds" answers "how much" without the number being what a later question shares — so neither
|
|
865
|
+
* is counted.
|
|
866
|
+
*/
|
|
867
|
+
/** Stems of a statement text. The text is the whole input, so a later write of a different statement is a different key. */
|
|
868
|
+
const STATEMENT_STEMS_MAX = 4096;
|
|
869
|
+
const statementStems = new Map();
|
|
870
|
+
const TERM_STEMS_MAX = 1024;
|
|
871
|
+
const termStems = new Map();
|
|
872
|
+
const QUESTION_TERMS_MAX = 64;
|
|
873
|
+
const questionTermCache = new Map();
|
|
874
|
+
function cachedQuestionTerms(query) {
|
|
875
|
+
const held = questionTermCache.get(query);
|
|
876
|
+
if (held)
|
|
877
|
+
return held;
|
|
878
|
+
const terms = questionTerms(query);
|
|
879
|
+
if (questionTermCache.size >= QUESTION_TERMS_MAX)
|
|
880
|
+
questionTermCache.delete(questionTermCache.keys().next().value);
|
|
881
|
+
questionTermCache.set(query, terms);
|
|
882
|
+
return terms;
|
|
883
|
+
}
|
|
884
|
+
function cachedTermStems(term) {
|
|
885
|
+
const held = termStems.get(term);
|
|
886
|
+
if (held)
|
|
887
|
+
return held;
|
|
888
|
+
const stems = wordStems([term]);
|
|
889
|
+
if (termStems.size >= TERM_STEMS_MAX)
|
|
890
|
+
termStems.delete(termStems.keys().next().value);
|
|
891
|
+
termStems.set(term, stems);
|
|
892
|
+
return stems;
|
|
893
|
+
}
|
|
894
|
+
function cachedStatementStems(text) {
|
|
895
|
+
const held = statementStems.get(text);
|
|
896
|
+
if (held)
|
|
897
|
+
return held;
|
|
898
|
+
const stems = new Set(wordStems((0, factTerms_1.atoms)(text)));
|
|
899
|
+
if (statementStems.size >= STATEMENT_STEMS_MAX)
|
|
900
|
+
statementStems.delete(statementStems.keys().next().value);
|
|
901
|
+
statementStems.set(text, stems);
|
|
902
|
+
return stems;
|
|
903
|
+
}
|
|
904
|
+
function statementRelevance(text, query) {
|
|
905
|
+
const terms = cachedQuestionTerms(query);
|
|
906
|
+
if (terms.length === 0)
|
|
907
|
+
return 0;
|
|
908
|
+
const stems = cachedStatementStems(text);
|
|
909
|
+
let held = 0;
|
|
910
|
+
for (const term of terms)
|
|
911
|
+
if (cachedTermStems(term).some(stem => stems.has(stem)))
|
|
912
|
+
held++;
|
|
913
|
+
return held;
|
|
914
|
+
}
|
|
915
|
+
/**
|
|
916
|
+
* The least of the question's content words a session statement must hold to be taken with the newest session's
|
|
917
|
+
* reservation, before the matches are taken by score (applyBudget `sessionStatements`). One shared word is a
|
|
918
|
+
* coincidence: a session header shares a date, a turn about something else shares a topic word, and the statement
|
|
919
|
+
* that decides the answer sometimes shares none at all. Two is the least that says a statement is about the
|
|
920
|
+
* question, so a statement below it keeps the expansion's own leftover room, in document order, which is where it
|
|
921
|
+
* was. Measured on the LongMemEval held-out knowledge-update cases: every statement the reservation delivered held
|
|
922
|
+
* two or more of the question's words, and the statements it displaced from other sessions held one or none.
|
|
923
|
+
*/
|
|
924
|
+
exports.STATEMENT_RELEVANCE_FLOOR = 2;
|
|
925
|
+
/**
|
|
926
|
+
* The statements of the session `memories[at]` belongs to, as positions in `memories`, ranked for the budget's
|
|
927
|
+
* reservation: most relevant to the question first (`statementRelevance`), and only from STATEMENT_RELEVANCE_FLOOR, so
|
|
928
|
+
* a statement the question shares fewer words with is a coincidence and keeps the expansion's own leftover room. Equal
|
|
929
|
+
* relevance keeps the order of the list, which is document order inside the note. A session that said something newer
|
|
930
|
+
* about the question said the deciding thing somewhere else, and the statement that holds it is usually not the first
|
|
931
|
+
* the session said: a header, a turn about something else, the answer. `floor` is the fewest shared words a statement
|
|
932
|
+
* needs: STATEMENT_RELEVANCE_FLOOR unless the caller ranks what it takes by relevance per token, as the budget's
|
|
933
|
+
* reservation does (NEWEST_SESSION_STATEMENT_FLOOR).
|
|
934
|
+
*/
|
|
935
|
+
function sessionStatementsFor(memories, at, query, floor = exports.STATEMENT_RELEVANCE_FLOOR) {
|
|
936
|
+
if (at < 0 || !query.trim())
|
|
937
|
+
return [];
|
|
938
|
+
const session = sessionOf(memories[at]);
|
|
939
|
+
return memories
|
|
940
|
+
.map((memory, i) => ({ memory, i }))
|
|
941
|
+
.filter(({ memory, i }) => i !== at && memory.reason === orchestrator_1.SESSION_EXPAND_REASON && sessionOf(memory) === session)
|
|
942
|
+
.map(({ memory, i }) => ({ i, relevance: statementRelevance(memory.text, query) }))
|
|
943
|
+
.filter(entry => entry.relevance >= floor)
|
|
944
|
+
.sort((a, b) => b.relevance - a.relevance)
|
|
945
|
+
.map(entry => entry.i);
|
|
946
|
+
}
|
|
947
|
+
/**
|
|
948
|
+
* The other matches of the session `memories[at]` belongs to that are themselves above the relevance bar (as `newestMatchAbove`
|
|
949
|
+
* counts), as positions in `memories`, in the order of the list: within a session that is best score first. What a session
|
|
950
|
+
* said about the question is rarely one chunk, and the chunk that holds the answer is often not its closest.
|
|
951
|
+
*/
|
|
952
|
+
function sessionMatesAbove(memories, similarities, bar, at) {
|
|
953
|
+
if (at < 0)
|
|
954
|
+
return [];
|
|
955
|
+
const session = sessionOf(memories[at]);
|
|
956
|
+
const mates = [];
|
|
957
|
+
memories.forEach((memory, i) => {
|
|
958
|
+
if (i === at || memory.reason === orchestrator_1.SESSION_EXPAND_REASON || !memory.session_date || sessionOf(memory) !== session)
|
|
959
|
+
return;
|
|
960
|
+
const heard = similarities.find(hit => isTopHit(hit, memory));
|
|
961
|
+
if (heard !== undefined && heard.similarity >= bar)
|
|
962
|
+
mates.push(i);
|
|
963
|
+
});
|
|
964
|
+
return mates;
|
|
965
|
+
}
|
|
966
|
+
function applyBudget(facts, factEvents, memories, memoryEvents, allowance, cost, plan = {}) {
|
|
967
|
+
const { keep, recent, relevance } = plan;
|
|
968
|
+
// `keep` comes before the reserved session when nothing is reserved, when it is a fact, or when it says more about the question.
|
|
969
|
+
const keepFirst = keep === undefined || recent === undefined || relevance === undefined || !memories.includes(keep) || relevance(keep) > relevance(recent);
|
|
970
|
+
const budgeted = budgetWith(keepFirst, facts, factEvents, memories, memoryEvents, allowance, cost, plan);
|
|
971
|
+
if (keepFirst)
|
|
972
|
+
return budgeted;
|
|
973
|
+
// The reserved session came first, and the facts took the room before it: if neither the match the verdict was judged on
|
|
974
|
+
// nor `recent` is in the reply, the verdict has nothing to stand on (`stands`) and the reply loses `relevance`. `keep`
|
|
975
|
+
// then takes its room first, and the session takes what is left in the order it always did (`ordered`): the room a held match
|
|
976
|
+
// leaves is too small to spend on a ranking by the words of the question.
|
|
977
|
+
const reserved = memories.indexOf(recent);
|
|
978
|
+
const holdsRecent = budgeted.memories.includes(recent)
|
|
979
|
+
|| (reserved >= 0 && memoryEvents[reserved].length > 0 && memoryEvents[reserved].every(id => budgeted.listed.has(id)));
|
|
980
|
+
const holdsKeep = budgeted.memories.includes(keep)
|
|
981
|
+
|| (memoryEvents[memories.indexOf(keep)]?.length > 0 && memoryEvents[memories.indexOf(keep)].every(id => budgeted.listed.has(id)));
|
|
982
|
+
return holdsRecent || holdsKeep ? budgeted : budgetWith(true, facts, factEvents, memories, memoryEvents, allowance, cost, plan, true);
|
|
983
|
+
}
|
|
984
|
+
function budgetWith(keepFirst, facts, factEvents, memories, memoryEvents, allowance, cost, { keep, closest, recent, mates = [], sessionStatements = [], relevance } = {}, ordered = false) {
|
|
985
|
+
const added = (memory) => memories[memory].reason === orchestrator_1.SESSION_EXPAND_REASON;
|
|
986
|
+
const factShare = memories.some((_, i) => !added(i)) ? Math.floor(allowance * exports.FACTS_BUDGET_SHARE) : allowance;
|
|
987
|
+
let used = 0;
|
|
988
|
+
let held; // `keep`, once it has its room
|
|
989
|
+
const fits = (item, limit) => {
|
|
990
|
+
if (item === held)
|
|
991
|
+
return true; // its room is taken
|
|
992
|
+
if (used + cost(item) > limit)
|
|
993
|
+
return false;
|
|
994
|
+
used += cost(item);
|
|
995
|
+
return true;
|
|
996
|
+
};
|
|
997
|
+
const listed = new Set();
|
|
998
|
+
const kept = new Set(); // the memories listed, by position
|
|
999
|
+
const repeats = (memory) => memoryEvents[memory].length > 0 && memoryEvents[memory].every(id => listed.has(id));
|
|
1000
|
+
// A fact the reply lists: a memory that is its event is not listed again, and gives its room back.
|
|
1001
|
+
const list = (event) => {
|
|
1002
|
+
if (!event)
|
|
1003
|
+
return;
|
|
1004
|
+
listed.add(event);
|
|
1005
|
+
for (const memory of kept) {
|
|
1006
|
+
if (repeats(memory)) {
|
|
1007
|
+
kept.delete(memory);
|
|
1008
|
+
used -= cost(memories[memory]);
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
};
|
|
1012
|
+
const placeKeep = () => {
|
|
1013
|
+
if (keep === undefined)
|
|
1014
|
+
return;
|
|
1015
|
+
const heldMemory = memories.indexOf(keep);
|
|
1016
|
+
if (heldMemory >= 0 && kept.has(heldMemory))
|
|
1017
|
+
held = keep; // the reservation took it, and its room is paid
|
|
1018
|
+
else if (fits(keep, allowance)) {
|
|
1019
|
+
held = keep;
|
|
1020
|
+
const heldFact = facts.indexOf(keep);
|
|
1021
|
+
if (heldFact >= 0)
|
|
1022
|
+
list(factEvents[heldFact]);
|
|
1023
|
+
else if (heldMemory >= 0)
|
|
1024
|
+
kept.add(heldMemory);
|
|
1025
|
+
else if (keep === closest?.fact)
|
|
1026
|
+
list(closest.event);
|
|
1027
|
+
}
|
|
1028
|
+
};
|
|
1029
|
+
if (keepFirst)
|
|
1030
|
+
placeKeep();
|
|
1031
|
+
// The top-ranked fact that the whole allowance cannot hold is delivered as a marked truncated prefix that fits
|
|
1032
|
+
// the room it has, not dropped: a long fact that is itself the answer is the one item the budget must not lose.
|
|
1033
|
+
// (The P2-1 cap is a crowding rule and never touches the top fact; this is the no-room case, where the choice is
|
|
1034
|
+
// a prefix or nothing.)
|
|
1035
|
+
let topFactCut = false;
|
|
1036
|
+
let top = facts;
|
|
1037
|
+
if (facts.length > 0 && used + cost(facts[0]) > allowance) {
|
|
1038
|
+
const first = facts[0];
|
|
1039
|
+
const object = first.object ?? '';
|
|
1040
|
+
const room = Math.max(1, allowance - used - (cost(first) - replyTokens(object)));
|
|
1041
|
+
top = [{ ...first, object: capFactObject(object, room) }, ...facts.slice(1)];
|
|
1042
|
+
topFactCut = true;
|
|
1043
|
+
}
|
|
1044
|
+
let keptFacts = 0;
|
|
1045
|
+
// The first fact is the best match, so it may take more than the facts' share.
|
|
1046
|
+
while (keptFacts < top.length && fits(top[keptFacts], keptFacts === 0 ? allowance : factShare))
|
|
1047
|
+
list(factEvents[keptFacts++]);
|
|
1048
|
+
// The newest match above the relevance bar and the rest of its session take their room before the others are taken by score.
|
|
1049
|
+
// They are one pool, taken by relevance to the question squared per token (a statement weighing more), at most
|
|
1050
|
+
// NEWEST_SESSION_PIECES of them in at most NEWEST_SESSION_SHARE of the budget. One that does not fit that room, or the
|
|
1051
|
+
// allowance, is passed over for the next (it must not end the run), so a session's closest chunk can be too big for the budget
|
|
1052
|
+
// while what it said elsewhere still fits, and the statement that decides the answer is usually one of the small ones.
|
|
1053
|
+
const reserved = recent === undefined ? -1 : memories.indexOf(recent);
|
|
1054
|
+
if (reserved >= 0) {
|
|
1055
|
+
const room = Math.floor(allowance * exports.NEWEST_SESSION_SHARE);
|
|
1056
|
+
const said = (item) => relevance?.(item) ?? 0;
|
|
1057
|
+
const weight = (item) => (item.reason === orchestrator_1.SESSION_EXPAND_REASON ? exports.NEWEST_SESSION_STATEMENT_WEIGHT : 1);
|
|
1058
|
+
const value = (item) => (weight(item) * said(item) * said(item)) / Math.max(1, cost(item));
|
|
1059
|
+
// `ordered`: the match first, then its other matches, then the statements that share two words of the question, as listed.
|
|
1060
|
+
const pool = ordered
|
|
1061
|
+
? [recent, ...mates, ...sessionStatements.filter(statement => said(statement) >= exports.STATEMENT_RELEVANCE_FLOOR)]
|
|
1062
|
+
: [recent, ...mates, ...sessionStatements].sort((a, b) => value(b) - value(a));
|
|
1063
|
+
// A small match cannot buy a share of the budget in unmatched statements (PR #520): the statements together cost no more
|
|
1064
|
+
// than the match itself. The match need not fit, so a large matching chunk can still reserve a short deciding statement.
|
|
1065
|
+
const statementRoom = cost(memories[reserved]);
|
|
1066
|
+
let statementSpent = 0;
|
|
1067
|
+
let spent = 0;
|
|
1068
|
+
let taken = 0;
|
|
1069
|
+
let weakTaken = 0;
|
|
1070
|
+
for (const piece of pool) {
|
|
1071
|
+
if (taken >= (ordered ? Infinity : exports.NEWEST_SESSION_PIECES))
|
|
1072
|
+
break;
|
|
1073
|
+
const statement = piece.reason === orchestrator_1.SESSION_EXPAND_REASON;
|
|
1074
|
+
const weak = !ordered && statement && said(piece) < 2;
|
|
1075
|
+
if (weak && weakTaken >= exports.NEWEST_SESSION_WEAK_STATEMENTS)
|
|
1076
|
+
continue;
|
|
1077
|
+
const i = memories.indexOf(piece);
|
|
1078
|
+
if (ordered && piece === recent) { // the match itself is no part of the session's room
|
|
1079
|
+
if (i >= 0 && !kept.has(i) && !repeats(i) && fits(piece, allowance))
|
|
1080
|
+
kept.add(i);
|
|
1081
|
+
continue;
|
|
1082
|
+
}
|
|
1083
|
+
if (i < 0 || kept.has(i) || repeats(i) || spent + cost(piece) > room || (statement && statementSpent + cost(piece) > statementRoom) || !fits(piece, allowance))
|
|
1084
|
+
continue;
|
|
1085
|
+
spent += cost(piece);
|
|
1086
|
+
if (statement)
|
|
1087
|
+
statementSpent += cost(piece);
|
|
1088
|
+
taken++;
|
|
1089
|
+
if (weak)
|
|
1090
|
+
weakTaken++;
|
|
1091
|
+
kept.add(i);
|
|
1092
|
+
}
|
|
1093
|
+
// Nothing of the session's matches was taken: the match itself is tried alone, as the reservation always did.
|
|
1094
|
+
const sessionKept = [...kept].some(i => !added(i) && sessionOf(memories[i]) === sessionOf(memories[reserved]));
|
|
1095
|
+
if (!sessionKept && !repeats(reserved) && fits(memories[reserved], allowance))
|
|
1096
|
+
kept.add(reserved);
|
|
1097
|
+
}
|
|
1098
|
+
if (!keepFirst)
|
|
1099
|
+
placeKeep();
|
|
1100
|
+
// A top-ranked match too big for the whole allowance is passed over, and then every later match that fits is taken, in score
|
|
1101
|
+
// order, as the top fact is cut (topFactCut). Before, one long note that topped a generic question (the report's "what does
|
|
1102
|
+
// ember need": 1,100 words above an 80-token note) emptied the reply at every budget that could not hold it. A top match that
|
|
1103
|
+
// fits the allowance is never passed over: while nothing is passed over, a later match that does not fit ends the run, so a
|
|
1104
|
+
// budget that holds the top match keeps the best matches (cut by score), not the next ones that happen to be small. The top
|
|
1105
|
+
// match is passed over only when it can never fit: a reply cut back from over budget must not swap it for a smaller one.
|
|
1106
|
+
const ranked = matchBudgetOrder(memories);
|
|
1107
|
+
const topMatch = ranked.find(i => !repeats(i) && !kept.has(i));
|
|
1108
|
+
let passedOver = false;
|
|
1109
|
+
for (const i of ranked) {
|
|
1110
|
+
if (repeats(i) || kept.has(i))
|
|
1111
|
+
continue;
|
|
1112
|
+
if (fits(memories[i], allowance)) {
|
|
1113
|
+
kept.add(i);
|
|
1114
|
+
continue;
|
|
1115
|
+
}
|
|
1116
|
+
if (i === topMatch && cost(memories[i]) > allowance)
|
|
1117
|
+
passedOver = true;
|
|
1118
|
+
else if (!passedOver)
|
|
1119
|
+
break;
|
|
1120
|
+
}
|
|
1121
|
+
while (keptFacts < top.length && fits(top[keptFacts], allowance))
|
|
1122
|
+
list(factEvents[keptFacts++]);
|
|
1123
|
+
// What the expansion added takes what the facts and the matches left, in the order it was laid out.
|
|
1124
|
+
for (let i = 0; i < memories.length; i++) {
|
|
1125
|
+
if (!added(i))
|
|
1126
|
+
continue;
|
|
1127
|
+
if (!fits(memories[i], allowance))
|
|
1128
|
+
break;
|
|
1129
|
+
kept.add(i);
|
|
1130
|
+
}
|
|
1131
|
+
const shownFacts = top.filter((fact, i) => i < keptFacts || fact === held);
|
|
1132
|
+
if (closest !== undefined && (closest.fact === held || (keptFacts === top.length && fits(closest.fact, allowance)))) {
|
|
1133
|
+
shownFacts.push(closest.fact);
|
|
1134
|
+
list(closest.event);
|
|
1135
|
+
}
|
|
1136
|
+
const shown = [...kept].sort((a, b) => a - b);
|
|
1137
|
+
return {
|
|
1138
|
+
facts: shownFacts,
|
|
1139
|
+
memories: shown.map(i => memories[i]),
|
|
1140
|
+
truncated: topFactCut || shownFacts.length < top.length + (closest === undefined ? 0 : 1) || memories.some((_, i) => !added(i) && !kept.has(i) && !repeats(i)),
|
|
1141
|
+
tokens: used,
|
|
1142
|
+
listed,
|
|
1143
|
+
};
|
|
1144
|
+
}
|
|
1145
|
+
/**
|
|
1146
|
+
* Client-safe note path: never leak a host path through a product surface
|
|
1147
|
+
* (the same rule the engine search tools follow). Paths inside the memory dir
|
|
1148
|
+
* render relative to it; anything else falls back to the basename.
|
|
1149
|
+
*/
|
|
1150
|
+
function clientSafePath(memoryDir, stored) {
|
|
1151
|
+
const resolved = path_1.default.resolve(stored);
|
|
1152
|
+
const root = path_1.default.resolve(memoryDir);
|
|
1153
|
+
if (resolved === root || resolved.startsWith(root + path_1.default.sep)) {
|
|
1154
|
+
return path_1.default.relative(root, resolved) || stored;
|
|
1155
|
+
}
|
|
1156
|
+
return path_1.default.basename(stored);
|
|
1157
|
+
}
|
|
1158
|
+
/**
|
|
1159
|
+
* The most of max_tokens a correction listed for a replaced memory may take. A correction can be as long as a statement
|
|
1160
|
+
* (4,000 characters), and a long item the budget cannot fit stops the run of matches behind it (applyBudget), so a
|
|
1161
|
+
* correction the replacement brings into a reply is cut to this share, with the marker that says where to read the rest, as
|
|
1162
|
+
* a long fact is.
|
|
1163
|
+
*/
|
|
1164
|
+
const CORRECTION_SHARE = 0.3;
|
|
1165
|
+
/**
|
|
1166
|
+
* The correction the budget works with in place of each one a replaced memory has: it takes the best score of the memories it
|
|
1167
|
+
* corrects, and as a match found only by its words it is ranked with the matches retrieval scored (reason "+replacement"),
|
|
1168
|
+
* so a correction that says it in other words than the memory it replaces is not cut before that memory. Returns the
|
|
1169
|
+
* corrections as they are budgeted, by the one they stand for, and the correction each replaced memory has.
|
|
1170
|
+
*/
|
|
1171
|
+
function bindCorrections(pairs, maxTokens) {
|
|
1172
|
+
const share = Math.max(32, Math.floor(maxTokens * CORRECTION_SHARE));
|
|
1173
|
+
const bound = new Map();
|
|
1174
|
+
for (const { replaced, correction } of pairs) {
|
|
1175
|
+
let item = bound.get(correction);
|
|
1176
|
+
if (!item) {
|
|
1177
|
+
const wordsOnly = correction.reason.split('+').every(reason => WORD_MATCH_REASONS.includes(reason));
|
|
1178
|
+
item = {
|
|
1179
|
+
...correction,
|
|
1180
|
+
...(replyTokens(correction.text) > share ? { text: capFactObject(correction.text, share) } : {}),
|
|
1181
|
+
...(wordsOnly ? { reason: `${correction.reason}+replacement` } : {}),
|
|
1182
|
+
};
|
|
1183
|
+
}
|
|
1184
|
+
const score = Math.max(item.score ?? Number.NEGATIVE_INFINITY, replaced.score ?? Number.NEGATIVE_INFINITY);
|
|
1185
|
+
bound.set(correction, Number.isFinite(score) ? { ...item, score } : item);
|
|
1186
|
+
}
|
|
1187
|
+
const correctionOf = new Map();
|
|
1188
|
+
for (const { replaced, correction } of pairs)
|
|
1189
|
+
correctionOf.set(replaced, bound.get(correction));
|
|
1190
|
+
return { bound, correctionOf };
|
|
1191
|
+
}
|
|
1192
|
+
const sameMemory = (a, b) => a.notePath === b.notePath && (a.headingPath ?? '') === (b.headingPath ?? '') && a.text === b.text;
|
|
1193
|
+
/**
|
|
1194
|
+
* A reply never lists a replaced memory before its correction, nor without it: each is moved to follow the correction, and
|
|
1195
|
+
* one whose correction the reply does not hold is left out (the reply says it cut something).
|
|
1196
|
+
*/
|
|
1197
|
+
function pinToCorrections(reply, correctionOf) {
|
|
1198
|
+
let listed = reply.memories;
|
|
1199
|
+
const replacedOf = (memory) => memory.replaced === undefined ? undefined : [...correctionOf].find(([replaced]) => sameMemory(replaced, memory))?.[1];
|
|
1200
|
+
let cut = false;
|
|
1201
|
+
for (const memory of [...listed]) {
|
|
1202
|
+
const correction = replacedOf(memory);
|
|
1203
|
+
if (correction === undefined)
|
|
1204
|
+
continue;
|
|
1205
|
+
const at = listed.findIndex(other => other !== memory && sameMemory(other, correction));
|
|
1206
|
+
const from = listed.indexOf(memory);
|
|
1207
|
+
if (at < 0) {
|
|
1208
|
+
listed = listed.filter(other => other !== memory);
|
|
1209
|
+
cut = true;
|
|
1210
|
+
}
|
|
1211
|
+
else if (at > from) {
|
|
1212
|
+
listed = [...listed.slice(0, from), ...listed.slice(from + 1, at + 1), memory, ...listed.slice(at + 1)];
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
reply.memories = listed;
|
|
1216
|
+
if (cut)
|
|
1217
|
+
reply.truncated = true;
|
|
1218
|
+
}
|
|
1219
|
+
async function buildAgentContext(db, engine, memoryDir, options = {}) {
|
|
1220
|
+
const query = (options.query ?? '').trim();
|
|
1221
|
+
const scopeNorm = options.scope === undefined ? undefined : (0, scope_1.normalizeScope)(options.scope);
|
|
1222
|
+
const maxTokens = Math.min(Math.max(options.maxTokens ?? 800, 128), 4000);
|
|
1223
|
+
let rankedBy;
|
|
1224
|
+
const memories = [];
|
|
1225
|
+
// What retrieval's session expansion added is no match; it is budgeted after every match (see applyBudget).
|
|
1226
|
+
const added = [];
|
|
1227
|
+
let unfinishedEvidence;
|
|
1228
|
+
let semanticAvailable = false;
|
|
1229
|
+
let keywordAvailable = false;
|
|
1230
|
+
let notice;
|
|
1231
|
+
// A search that could not run (not one that found nothing), named so an empty reply is never mistaken for an empty store.
|
|
1232
|
+
const degraded = [];
|
|
1233
|
+
let keywordFailed = false;
|
|
1234
|
+
const probe = {};
|
|
1235
|
+
if (query) {
|
|
1236
|
+
// Preserve the uncapped get_context default; an explicit environment cap still wins.
|
|
1237
|
+
const semantic = await (0, engine_1.engineSearchMemory)(db, engine, memoryDir, query, 8, scopeNorm, options.evidence, options.includeHistory === true, options.ledgerRelPath, probe, undefined, 0);
|
|
1238
|
+
semanticAvailable = semantic.ok;
|
|
1239
|
+
notice = semantic.notice; // semantic search is off while the embeddings are rebuilt, or the provider failed it: say so
|
|
1240
|
+
if (semantic.error === 'semantic-unavailable' && semantic.notice)
|
|
1241
|
+
degraded.push('semantic');
|
|
1242
|
+
if (semantic.ok) {
|
|
1243
|
+
rankedBy = semantic.rankedBy;
|
|
1244
|
+
for (const hit of semantic.hits) {
|
|
1245
|
+
const normalized = normalizeMemory(hit, 'semantic', options.reader);
|
|
1246
|
+
if (normalized)
|
|
1247
|
+
(normalized.reason === orchestrator_1.SESSION_EXPAND_REASON ? added : memories).push(normalized);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
// Bounded-wait evidence (RENOR_EVIDENCE_ASYNC) can answer before the table exists. get_context then says so in the
|
|
1251
|
+
// words search_memory uses; with the flag off neither reason occurs and the response is what it was.
|
|
1252
|
+
if (semantic.evidence?.reason === 'pending' || semantic.evidence?.reason === 'queue-full' || semantic.evidence?.reason === 'daily-cap') {
|
|
1253
|
+
const { state, note } = (0, attach_1.productEvidenceReport)(semantic.evidence);
|
|
1254
|
+
unfinishedEvidence = { state, note };
|
|
1255
|
+
}
|
|
1256
|
+
// The variants find the same chunks of the ledger: each chunk is parsed once for this read.
|
|
1257
|
+
const ledgerRecords = new Map();
|
|
1258
|
+
(0, readCache_1.inReadSnapshot)(db, () => {
|
|
1259
|
+
for (const variant of keywordVariants(query)) {
|
|
1260
|
+
const keyword = (0, engine_1.engineSearchText)(db, memoryDir, variant, 6, scopeNorm, options.includeHistory === true, options.ledgerRelPath, ledgerRecords);
|
|
1261
|
+
keywordAvailable = keywordAvailable || keyword.ok;
|
|
1262
|
+
if (!keyword.ok) {
|
|
1263
|
+
keywordFailed = true;
|
|
1264
|
+
continue;
|
|
1265
|
+
}
|
|
1266
|
+
for (const hit of keyword.results) {
|
|
1267
|
+
const normalized = normalizeMemory(hit, 'fts', options.reader);
|
|
1268
|
+
if (normalized)
|
|
1269
|
+
memories.push(normalized);
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
});
|
|
1273
|
+
if (keywordFailed && !keywordAvailable) {
|
|
1274
|
+
degraded.push('keyword');
|
|
1275
|
+
notice = `${notice ? `${notice.replace('the saved facts and keyword search do not need it and still answer.', 'the saved facts still answer.')} ` : ''}`
|
|
1276
|
+
+ 'Keyword search is unavailable: its index is missing, so nothing here was found by words.';
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
// A permission question also looks for the standing facts by meaning, through the embedding path the search above
|
|
1280
|
+
// used (and its query vector, when it ran), when that path is the calibrated (Vertex) model; otherwise, or when it
|
|
1281
|
+
// fails, by words alone (permissionMeaning).
|
|
1282
|
+
const meaning = query ? await permissionMeaning(db, engine, query, scopeNorm, probe.queryVector) : undefined;
|
|
1283
|
+
const facts = (0, readCache_1.inReadSnapshot)(db, () => currentFactMatches(db, query, scopeNorm, query ? 12 : 20, meaning));
|
|
1284
|
+
// After the keyword matches, so a statement a keyword search also found is that match, not the added copy.
|
|
1285
|
+
const merged = dedupeMemories([...memories, ...added]);
|
|
1286
|
+
// A correction takes the best score of the memories it corrects and is cut to a share of the budget (`bindCorrections`), and
|
|
1287
|
+
// the reply keeps them in pairs: a replaced memory is never listed without its correction, nor before it, whatever the
|
|
1288
|
+
// budget cut and however the layout sorts (`pinToCorrections`, and `pinned` below). Awaited only when the workspace has a
|
|
1289
|
+
// replacement: without one the assembly below stays synchronous, as it always was.
|
|
1290
|
+
const pairs = [];
|
|
1291
|
+
const arranged = (0, replacements_1.hasReplacements)(memoryDir)
|
|
1292
|
+
? await (0, replacements_1.withReplacements)({ db, memoryDir, ledgerRelPath: options.ledgerRelPath ?? lifecycle_1.DEFAULT_LEDGER_REL, indexOptions: {} }, merged, { pairs })
|
|
1293
|
+
: merged;
|
|
1294
|
+
const { bound, correctionOf: correctionOfRaw } = pairs.length > 0 ? bindCorrections(pairs, maxTokens) : { bound: new Map(), correctionOf: new Map() };
|
|
1295
|
+
const deduped = arranged.map(memory => bound.get(memory) ?? memory);
|
|
1296
|
+
const hasMemories = deduped.some(m => m.reason !== orchestrator_1.SESSION_EXPAND_REASON);
|
|
1297
|
+
// Relevance signal (RENOR_CONTEXT_RELEVANCE): absent when off. Judged before the budget, and the response holds the
|
|
1298
|
+
// match it reports. A fact close in meaning that the words of the request did not find ("Where am I based?" for
|
|
1299
|
+
// home_city) stands in when they found none; otherwise it follows the facts they found, in the room left, unless it is
|
|
1300
|
+
// that match: then, like a note chunk that is, the budget makes room for it.
|
|
1301
|
+
const judged = query && (0, contextRelevance_1.contextRelevanceEnabled)(options.env)
|
|
1302
|
+
? (0, readCache_1.inReadSnapshot)(db, () => (0, contextRelevance_1.assessQueryRelevance)({
|
|
1303
|
+
db, engine, scope: scopeNorm, probe, env: options.env, components: options.relevanceComponents,
|
|
1304
|
+
query, onFile: nameOnFile(db, memoryDir, { scope: scopeNorm, includeHistory: options.includeHistory === true, ledgerRelPath: options.ledgerRelPath }),
|
|
1305
|
+
}))
|
|
1306
|
+
: null;
|
|
1307
|
+
const closestFact = judged?.closestFact;
|
|
1308
|
+
const shownFacts = facts.length === 0 && closestFact ? [closestFact] : facts;
|
|
1309
|
+
// The values another writer replaced that stay open against a shown fact (memory/writerConflicts.ts): a fact with one
|
|
1310
|
+
// carries its line, and a fact with none is shaped as it always was.
|
|
1311
|
+
const disputes = (0, writerConflicts_1.writerConflictLines)(db, shownFacts.map(fact => fact.eventId));
|
|
1312
|
+
const shaped = (fact) => {
|
|
1313
|
+
const base = toAgentFact(fact, options.reader);
|
|
1314
|
+
const line = fact.eventId === null ? undefined : disputes.get(fact.eventId);
|
|
1315
|
+
return line === undefined ? base : { ...base, writerConflict: line };
|
|
1316
|
+
};
|
|
1317
|
+
const factLaneBudget = hasMemories ? Math.floor(maxTokens * exports.FACTS_BUDGET_SHARE) : maxTokens;
|
|
1318
|
+
const maxFactObjectTokens = Math.max(32, Math.floor(factLaneBudget * exports.MAX_FACT_OBJECT_LANE_SHARE));
|
|
1319
|
+
// The cap is a crowding rule: it applies only when the facts would not otherwise fit the lane, and never to the
|
|
1320
|
+
// top-ranked fact — a long fact that is itself the answer is delivered whole when the lane has room for it.
|
|
1321
|
+
const factCost = (fact) => replyTokens(JSON.stringify(shaped(fact))) + 1;
|
|
1322
|
+
const crowding = shownFacts.reduce((sum, fact) => sum + factCost(fact), 0) > factLaneBudget;
|
|
1323
|
+
let factObjectCut = false;
|
|
1324
|
+
const capFact = (fact, index) => {
|
|
1325
|
+
const object = fact.object ?? '';
|
|
1326
|
+
// A standing rule the permission lookup put first leads the page, but it is not necessarily the answer to the
|
|
1327
|
+
// action: a long one must not eat the lane and displace the fact the question's words found. So it is capped like
|
|
1328
|
+
// any non-top fact when another fact shares the page (review P3, 2026-10-09); a lone governance fact is whole.
|
|
1329
|
+
const leads = index === 0 && shownFacts.length > 1 && STANDING.has(fact);
|
|
1330
|
+
if ((crowding && index > 0 || leads) && replyTokens(object) > maxFactObjectTokens) {
|
|
1331
|
+
factObjectCut = true;
|
|
1332
|
+
return { ...shaped(fact), object: capFactObject(object, maxFactObjectTokens) };
|
|
1333
|
+
}
|
|
1334
|
+
return shaped(fact);
|
|
1335
|
+
};
|
|
1336
|
+
// The closest fact is the top-ranked one when the words found none (shownFacts is [closestFact]); otherwise it
|
|
1337
|
+
// follows the facts they found and is capped with them when the lane is crowded.
|
|
1338
|
+
const closestIndex = closestFact === undefined ? -1 : shownFacts.indexOf(closestFact);
|
|
1339
|
+
const closest = closestFact ? capFact(closestFact, closestIndex >= 0 ? closestIndex : shownFacts.length) : undefined;
|
|
1340
|
+
const isClosest = (fact) => closest !== undefined && fact.subject === closest.subject
|
|
1341
|
+
&& fact.predicate === closest.predicate && fact.scope === closest.scope && fact.validFrom === closest.validFrom
|
|
1342
|
+
&& objectIs(closest.object, fact.object);
|
|
1343
|
+
// A memory of the ledger names no note and no heading (retrieval/ledgerPresentation.ts); the event in its heading is
|
|
1344
|
+
// read first, so that a memory that is the event of a listed fact is not listed again. Its heading alone: its text is
|
|
1345
|
+
// the user's words now, which may hold a line that looks like a record's heading.
|
|
1346
|
+
const isLedger = (0, ledgerPresentation_1.ledgerNoteMatcher)({ memoryDir, ledgerRelPath: options.ledgerRelPath ?? lifecycle_1.DEFAULT_LEDGER_REL });
|
|
1347
|
+
const candidateFacts = shownFacts.map((fact, index) => capFact(fact, index));
|
|
1348
|
+
const factEvents = shownFacts.map(fact => fact.eventId?.toLowerCase() ?? null);
|
|
1349
|
+
// The agent-facing text: hidden spans labelled (hiddenSpans.ts). It is taken after every judgment above, which compares
|
|
1350
|
+
// the raw text (isTopHit), and before the budget, which counts what the reply will carry.
|
|
1351
|
+
// A ledger event that is a saved fact says who saved it, as get_context's facts do: its author is read from the fact.
|
|
1352
|
+
const eventAuthors = (0, factMarkers_1.factAuthorsByEvent)(db, deduped.flatMap(memory => (isLedger(memory.notePath) ? (0, ledgerLiveness_1.ledgerEventIds)(memory) : [])));
|
|
1353
|
+
const candidateMemories = deduped.map(memory => {
|
|
1354
|
+
if (!isLedger(memory.notePath))
|
|
1355
|
+
return (0, hiddenSpans_1.labelMemoryText)(memory);
|
|
1356
|
+
const plain = normalizeMemory({ ...memory, notePath: '', headingPath: '' }, memory.reason);
|
|
1357
|
+
return (0, hiddenSpans_1.labelMemoryText)({ ...plain, ...(0, factMarkers_1.eventMarker)((0, ledgerLiveness_1.ledgerEventIds)(memory), eventAuthors, options.reader) });
|
|
1358
|
+
});
|
|
1359
|
+
// The pairs were made on the memories as retrieved; the budget and the reply hold the labelled copies, so the pairs are
|
|
1360
|
+
// re-keyed to those (a memory with hidden text is a new object with different text).
|
|
1361
|
+
const listedAs = new Map(deduped.map((memory, i) => [memory, candidateMemories[i]]));
|
|
1362
|
+
const correctionOf = new Map([...correctionOfRaw].map(([replaced, correction]) => [listedAs.get(replaced) ?? replaced, listedAs.get(correction) ?? correction]));
|
|
1363
|
+
const memoryEvents = deduped.map(memory => (isLedger(memory.notePath) ? (0, ledgerLiveness_1.ledgerEventIds)({ text: '', headingPath: memory.headingPath }) : []));
|
|
1364
|
+
const missed = closest !== undefined && !candidateFacts.some(isClosest)
|
|
1365
|
+
? { fact: closest, event: closestFact?.eventId?.toLowerCase() ?? null }
|
|
1366
|
+
: undefined;
|
|
1367
|
+
// The match the verdict reports: the closest fact, or the note chunk the semantic search scored best. The probe records
|
|
1368
|
+
// that chunk with the paths the memories have before the ledger's are taken off, so it is found among those.
|
|
1369
|
+
const topMemory = judged?.matchedOn === 'chunk' ? deduped.findIndex(memory => isTopHit(probe.topHit, memory)) : -1;
|
|
1370
|
+
const match = judged?.matchedOn === 'fact'
|
|
1371
|
+
? candidateFacts.find(isClosest) ?? missed?.fact
|
|
1372
|
+
: topMemory >= 0
|
|
1373
|
+
? candidateMemories[topMemory]
|
|
1374
|
+
: undefined;
|
|
1375
|
+
// The newest match above the relevance bar keeps its place in the budget (applyBudget `recent`), where the layout is by
|
|
1376
|
+
// date (a read about the past is laid out by score and asks for the older statement) and the model has a bar to judge by,
|
|
1377
|
+
// and so does the rest of its session that is above the bar, in NEWEST_SESSION_SHARE of the budget (`mates`), and so do the statements
|
|
1378
|
+
// its expansion added to that session (`sessionStatementsFor` ranks them): the session said the deciding thing somewhere
|
|
1379
|
+
// else, and a statement is not scored, so the words of the question rank it.
|
|
1380
|
+
const bar = (0, contextRelevance_1.resolveRelevanceBar)(engine.modelKey, options.env);
|
|
1381
|
+
const newest = rankedBy === 'session_date' && bar !== null ? newestMatchAbove(deduped, probe.similarities ?? [], bar, query) : -1;
|
|
1382
|
+
const recent = newest >= 0 ? candidateMemories[newest] : undefined;
|
|
1383
|
+
const mates = bar === null ? [] : sessionMatesAbove(deduped, probe.similarities ?? [], bar, newest).map(i => candidateMemories[i]);
|
|
1384
|
+
const sessionStatements = sessionStatementsFor(deduped, newest, query, exports.NEWEST_SESSION_STATEMENT_FLOOR).map(i => candidateMemories[i]);
|
|
1385
|
+
// Whether the reply `kept` holds it: listed, or a memory that is the event of a fact it lists (that one is said once).
|
|
1386
|
+
// A fact the budget truncated to a prefix is still the match it was: same key, and its object is the prefix of the
|
|
1387
|
+
// candidate's (`objectIs` reads that direction), so the verdict a truncated top fact answers is still reported.
|
|
1388
|
+
const holds = (kept) => match !== undefined && (kept.facts.some(f => f === match || (f.subject === match.subject
|
|
1389
|
+
&& f.predicate === match.predicate
|
|
1390
|
+
&& f.scope === match.scope
|
|
1391
|
+
&& f.validFrom === match.validFrom
|
|
1392
|
+
&& objectIs(f.object, match.object)))
|
|
1393
|
+
|| kept.memories.includes(match)
|
|
1394
|
+
|| (topMemory >= 0 && match === candidateMemories[topMemory] && memoryEvents[topMemory].length > 0
|
|
1395
|
+
&& memoryEvents[topMemory].every(id => kept.listed.has(id))));
|
|
1396
|
+
// The verdict stands when the reply holds its match, or, for a note chunk, the newest match above the same bar that the
|
|
1397
|
+
// budget reserved in its place (`recent`): the reply then holds a match as relevant as the one the verdict judged, and the
|
|
1398
|
+
// verdict reports no more than that (`relevant`, the closest cosine and the bar). A fact the verdict names stays strict.
|
|
1399
|
+
const stands = (kept) => holds(kept) || (topMemory >= 0 && recent !== undefined
|
|
1400
|
+
&& (kept.memories.includes(recent) || (memoryEvents[newest].length > 0 && memoryEvents[newest].every(id => kept.listed.has(id)))));
|
|
1401
|
+
const retrieval = !query
|
|
1402
|
+
? 'facts'
|
|
1403
|
+
: semanticAvailable && keywordAvailable
|
|
1404
|
+
? 'hybrid'
|
|
1405
|
+
: semanticAvailable
|
|
1406
|
+
? 'semantic'
|
|
1407
|
+
: keywordAvailable
|
|
1408
|
+
? 'keyword'
|
|
1409
|
+
: 'facts';
|
|
1410
|
+
const trailer = (options.trailer && (0, readCache_1.inReadSnapshot)(db, options.trailer)) ?? {};
|
|
1411
|
+
// The facts this read could list: the ones its words found, and the closest one when they did not find it.
|
|
1412
|
+
const named = missed ? [...candidateFacts, missed.fact] : candidateFacts;
|
|
1413
|
+
// The opt-in read-honesty lane (LANE 3, RENOR_READ_HONESTY, off by default), set up once per read when it is armed.
|
|
1414
|
+
const honestyLane = (0, readHonesty_1.readHonestyEnabled)(options.env) && Boolean(query);
|
|
1415
|
+
// Lean memory items (RENOR_CONTEXT_LEAN_ITEMS, off by default): each memory is costed, and the reply measured, as it is
|
|
1416
|
+
// listed (leanMemory), so the same max_tokens holds more of them. The read-honesty lane lists provenance on every item,
|
|
1417
|
+
// so it keeps its own items.
|
|
1418
|
+
const lean = !honestyLane && contextLeanItemsEnabled(options.env);
|
|
1419
|
+
const listedTokens = (body) => replyTokens(JSON.stringify(lean ? { ...body, memories: body.memories.map(leanMemory) } : body));
|
|
1420
|
+
const ledgerRel = options.ledgerRelPath ?? lifecycle_1.DEFAULT_LEDGER_REL;
|
|
1421
|
+
const paths = honestyLane ? factSourcePaths(db, scopeNorm) : new Map();
|
|
1422
|
+
// Each item the lane lists says where it came from and when it was last verified.
|
|
1423
|
+
const factWithProvenance = (f) => {
|
|
1424
|
+
const sourcePath = paths.get(`${f.subject}\u0000${f.predicate}\u0000${f.scope ?? 'global'}`);
|
|
1425
|
+
return {
|
|
1426
|
+
...f,
|
|
1427
|
+
provenance: sourcePath && sourcePath !== LIFECYCLE_NOTE_PATH
|
|
1428
|
+
? (0, readHonesty_1.noteProvenance)(clientSafePath(memoryDir, sourcePath), f.validFrom)
|
|
1429
|
+
: (0, readHonesty_1.factProvenance)(f.validFrom, ledgerRel),
|
|
1430
|
+
};
|
|
1431
|
+
};
|
|
1432
|
+
const memoryWithProvenance = (m) => ({
|
|
1433
|
+
...m,
|
|
1434
|
+
provenance: (0, readHonesty_1.noteProvenance)(m.notePath, m.session_date ?? null),
|
|
1435
|
+
});
|
|
1436
|
+
// Conflicts are about the product's OWN stored facts, so their source paths are rendered client-safe too: a product
|
|
1437
|
+
// surface never leaks a host path (the same rule engineSearchMemory follows). A conflict across scopes is built again
|
|
1438
|
+
// from its rendered sides.
|
|
1439
|
+
const safe = (side) => ({ ...side, source: clientSafePath(memoryDir, side.source) });
|
|
1440
|
+
const clientSafe = (c) => ((0, readHonesty_1.acrossScopes)(c)
|
|
1441
|
+
? (0, readHonesty_1.conflictAcrossScopes)(c.subject, c.predicate, [c.newer, c.older, ...(c.alsoCurrent ?? [])].map(safe), c.total)
|
|
1442
|
+
: {
|
|
1443
|
+
...c,
|
|
1444
|
+
newer: { ...c.newer, source: clientSafePath(memoryDir, c.newer.source) },
|
|
1445
|
+
older: { ...c.older, source: clientSafePath(memoryDir, c.older.source) },
|
|
1446
|
+
statement: c.statement
|
|
1447
|
+
.split(c.newer.source).join(clientSafePath(memoryDir, c.newer.source))
|
|
1448
|
+
.split(c.older.source).join(clientSafePath(memoryDir, c.older.source)),
|
|
1449
|
+
});
|
|
1450
|
+
// The lane's contradictions for a reply listing `listed`: about what those facts say, in any scope the read sees, and
|
|
1451
|
+
// inside one scope about their subjects and the subjects the query names. Read once per list.
|
|
1452
|
+
let lastRead;
|
|
1453
|
+
const contradictions = (listed) => {
|
|
1454
|
+
if (lastRead?.listed !== listed) {
|
|
1455
|
+
const { conflicts } = (0, readHonesty_1.applyReadHonesty)({
|
|
1456
|
+
db, query, ledgerRelPath: ledgerRel, scope: scopeNorm, subjects: listed.map(f => f.subject), facts: listed, texts: [], env: options.env,
|
|
1457
|
+
});
|
|
1458
|
+
lastRead = { listed, found: conflicts.map(clientSafe) };
|
|
1459
|
+
}
|
|
1460
|
+
return lastRead.found;
|
|
1461
|
+
};
|
|
1462
|
+
const about = (fact, conflict) => (0, readHonesty_1.questionKey)(fact.subject, fact.predicate) === (0, readHonesty_1.questionKey)(conflict.subject, conflict.predicate);
|
|
1463
|
+
// What the lane judges is the read, not the reply: everything it could list, the facts it found and the memories a
|
|
1464
|
+
// reply listing all of them would list. "No good answer on file for this" is a claim about the store, so it is the
|
|
1465
|
+
// same at every max_tokens: a reply the budget cut down to one fact, or to none, does not get to make it while the
|
|
1466
|
+
// store covers the question, and a contradiction the read could name stands it down, as in applyReadHonesty.
|
|
1467
|
+
const judgedRead = honestyLane ? (() => {
|
|
1468
|
+
const factEventIds = new Set([...factEvents, missed?.event ?? null].filter((id) => id !== null));
|
|
1469
|
+
const listable = candidateMemories.filter((_, i) => !(memoryEvents[i].length > 0 && memoryEvents[i].every(id => factEventIds.has(id))));
|
|
1470
|
+
return (0, readHonesty_1.applyReadHonesty)({
|
|
1471
|
+
db,
|
|
1472
|
+
query,
|
|
1473
|
+
ledgerRelPath: ledgerRel,
|
|
1474
|
+
scope: scopeNorm,
|
|
1475
|
+
subjects: named.map(f => f.subject),
|
|
1476
|
+
facts: named,
|
|
1477
|
+
texts: [...named.map(f => `${f.subject} ${f.predicate} ${f.object}`), ...listable.map(m => m.text)],
|
|
1478
|
+
considered: named.length + listable.length,
|
|
1479
|
+
env: options.env,
|
|
1480
|
+
});
|
|
1481
|
+
})() : undefined;
|
|
1482
|
+
// A conflict-free superset cannot gain a conflict when facts are cut from it. Keep
|
|
1483
|
+
// that proof only for this synchronous assembly and the same committed DB contents.
|
|
1484
|
+
const conflictStamp = () => db.inTransaction ? null :
|
|
1485
|
+
`${(0, statements_1.prepareOnce)(db, 'PRAGMA data_version').pluck().get()}:${(0, statements_1.prepareOnce)(db, 'SELECT total_changes()').pluck().get()}`;
|
|
1486
|
+
let emptyConflictsAt = null;
|
|
1487
|
+
// The reply for the facts and memories `kept`. `relevant: true` reports its closest match, so it stands only when the
|
|
1488
|
+
// reply holds it (the match can be too big for max_tokens on its own); "not relevant" is about everything on file,
|
|
1489
|
+
// whatever is shown.
|
|
1490
|
+
const assemble = (kept, verdict = judged && (!judged.relevance.relevant || stands(kept)) ? judged.relevance : undefined) => {
|
|
1491
|
+
const body = {
|
|
1492
|
+
ok: true,
|
|
1493
|
+
query: query || null,
|
|
1494
|
+
scope: scopeNorm ?? 'all',
|
|
1495
|
+
retrieval,
|
|
1496
|
+
facts: kept.facts,
|
|
1497
|
+
// The order of the semantic memories (keyword matches, reason "fts" or "note-identity", follow them). Named
|
|
1498
|
+
// right above the list it describes: memories[0] is the newest match under "session_date", not the best one.
|
|
1499
|
+
...(rankedBy ? { rankedBy } : {}),
|
|
1500
|
+
memories: kept.memories,
|
|
1501
|
+
// A fact object the lane cut is a cut the reply says it made, whatever the budget did: the agent reads the
|
|
1502
|
+
// rest of that fact with read_memory or lookup_entity (the object itself carries TRUNCATION_MARKER).
|
|
1503
|
+
truncated: kept.truncated || factObjectCut,
|
|
1504
|
+
maxTokens,
|
|
1505
|
+
...(notice ? { notice } : {}),
|
|
1506
|
+
...(degraded.length > 0 ? { degraded } : {}),
|
|
1507
|
+
};
|
|
1508
|
+
if (verdict)
|
|
1509
|
+
body.relevance = verdict;
|
|
1510
|
+
// Multi-agent drift (memory/factConflicts.ts): a returned fact whose predicate has another current value under the
|
|
1511
|
+
// same subject in another scope, or under a related name, is named with those values. Absent when nothing
|
|
1512
|
+
// disagrees; within the read's scope only, so a scoped read never learns what another scope holds. Counted in
|
|
1513
|
+
// max_tokens with the fact that names it. The opt-in read-honesty lane below replaces it with its own `conflicts`
|
|
1514
|
+
// when armed.
|
|
1515
|
+
const conflicts = kept.facts.length > 0 && emptyConflictsAt !== null && emptyConflictsAt === conflictStamp()
|
|
1516
|
+
? [] : (0, factConflicts_1.namedConflicts)(db, kept.facts, scopeNorm ?? null, options.reader);
|
|
1517
|
+
if (conflicts.length > 0)
|
|
1518
|
+
body.conflicts = conflicts;
|
|
1519
|
+
// LANE 3 read-time honesty (RENOR_READ_HONESTY, default OFF). When off, the
|
|
1520
|
+
// response is byte-identical to the shipped contract — no new fields, no new
|
|
1521
|
+
// queries. When on: every item says where it came from and when it was last
|
|
1522
|
+
// verified, contradictions between current facts are named instead of
|
|
1523
|
+
// silently returned as compatible, and a question the stored evidence does
|
|
1524
|
+
// not cover gets an explicit "no good answer on file for this".
|
|
1525
|
+
if (honestyLane) {
|
|
1526
|
+
const listed = body.facts;
|
|
1527
|
+
body.facts = listed.map(factWithProvenance);
|
|
1528
|
+
body.memories = kept.memories.map(memoryWithProvenance);
|
|
1529
|
+
body.provenance = {
|
|
1530
|
+
ledger: ledgerRel,
|
|
1531
|
+
note: 'lifecycle facts are ledger-backed; note items are sourced from the note path shown on each item.',
|
|
1532
|
+
};
|
|
1533
|
+
// A contradiction about what a listed fact says is in the reply with it, at its least (#331's cut, readHonesty.ts
|
|
1534
|
+
// fitConflicts): the two values that disagree, and how many there are. The fact was charged that much (below), so
|
|
1535
|
+
// the two are inside max_tokens together or neither is. The lane's other contradictions, and more sides of these,
|
|
1536
|
+
// take the room the items leave (`withLane`, below).
|
|
1537
|
+
body.conflicts = (0, readHonesty_1.fitConflicts)(contradictions(listed).filter(c => listed.some(f => about(f, c))), 0);
|
|
1538
|
+
body.abstention = judgedRead.abstention;
|
|
1539
|
+
}
|
|
1540
|
+
if (unfinishedEvidence)
|
|
1541
|
+
body.evidence = unfinishedEvidence;
|
|
1542
|
+
return Object.assign(body, trailer);
|
|
1543
|
+
};
|
|
1544
|
+
// The whole reply is bounded by max_tokens, as the engine counts tokens (replyTokens): the echoed query, `relevance`,
|
|
1545
|
+
// `conflicts` and every other field count, not the facts and memories alone (live dogfood, 2026-10-03, D9: 19 of 31
|
|
1546
|
+
// replies were over, by up to 1.89 times). The items get what the rest of an empty reply leaves, a fact costing the
|
|
1547
|
+
// conflicts it is the first to name as well (it brings them into the reply). The reply they make is then measured
|
|
1548
|
+
// whole, and while it is over (by how separately counted pieces join), the items get that much less. Each round
|
|
1549
|
+
// keeps fewer tokens of items, so it ends; a reply whose fixed part alone is over (an echoed query longer than the
|
|
1550
|
+
// budget) holds no fact and no memory.
|
|
1551
|
+
//
|
|
1552
|
+
// The opt-in read-honesty lane (RENOR_READ_HONESTY, off by default) reads under the same bound, and what a budget cuts
|
|
1553
|
+
// is the least of what it says. Its items are counted as it lists them, with their provenance. A fact is charged the
|
|
1554
|
+
// contradiction about what it says at its least, the two values that disagree and how many there are, so the two are
|
|
1555
|
+
// inside max_tokens together or neither is. What else the lane can add takes the room the items leave (`withLane`).
|
|
1556
|
+
// A fact is charged the contradiction it names by its key (subject/predicate/scope), not its object: a fact the budget
|
|
1557
|
+
// truncated to a prefix is a new object, and it must carry the same charge as the candidate it was cut from, so a kept
|
|
1558
|
+
// fact and the contradiction it brings are still in the reply together or neither is.
|
|
1559
|
+
const bringsKey = (item) => 'predicate' in item ? `${item.subject}\u0000${item.predicate}\u0000${item.scope ?? 'global'}` : '';
|
|
1560
|
+
const brings = new Map();
|
|
1561
|
+
if (!honestyLane) { // that lane names its own contradictions instead
|
|
1562
|
+
const beforeConflicts = named.length > 0 ? conflictStamp() : null;
|
|
1563
|
+
const allConflicts = (0, factConflicts_1.namedConflicts)(db, named, scopeNorm ?? null, options.reader);
|
|
1564
|
+
if (allConflicts.length === 0)
|
|
1565
|
+
emptyConflictsAt = beforeConflicts;
|
|
1566
|
+
for (const conflict of allConflicts) {
|
|
1567
|
+
const first = named.find(f => f.predicate === conflict.predicate
|
|
1568
|
+
&& conflict.values.some(v => v.subject === f.subject && objectIs(f.object, v.object) && v.scope === f.scope))
|
|
1569
|
+
?? named.find(f => f.predicate === conflict.predicate);
|
|
1570
|
+
if (first)
|
|
1571
|
+
brings.set(bringsKey(first), (brings.get(bringsKey(first)) ?? 0) + replyTokens(JSON.stringify(conflict)) + 1);
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
else {
|
|
1575
|
+
// Over every fact the read could list, so the charge does not move as the list is cut; the first fact that says what
|
|
1576
|
+
// a contradiction is about pays for it, as the reply renders it (and, like the lane's items, without a comma).
|
|
1577
|
+
for (const conflict of judgedRead.conflicts.map(clientSafe)) {
|
|
1578
|
+
const first = named.find(f => about(f, conflict));
|
|
1579
|
+
if (first)
|
|
1580
|
+
brings.set(bringsKey(first), (brings.get(bringsKey(first)) ?? 0) + replyTokens(JSON.stringify((0, readHonesty_1.fitConflicts)([conflict], 0)[0])));
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1583
|
+
const costs = new Map();
|
|
1584
|
+
const cost = (item) => {
|
|
1585
|
+
let size = costs.get(item);
|
|
1586
|
+
if (size === undefined) {
|
|
1587
|
+
// Counted alone and with its comma, an item is a token or two over what it adds to the reply (`},{"` is one token
|
|
1588
|
+
// there): the items rarely add up to more than the reply they make, so a reply is rarely cut a second time. The
|
|
1589
|
+
// read-honesty lane counts an item as it lists it, with its provenance, and without the comma: what it adds to the
|
|
1590
|
+
// reply, so a fact and the contradiction it brings are listed from the least max_tokens that holds them.
|
|
1591
|
+
size = honestyLane
|
|
1592
|
+
? replyTokens(JSON.stringify('predicate' in item ? factWithProvenance(item) : memoryWithProvenance(item))) + (brings.get(bringsKey(item)) ?? 0)
|
|
1593
|
+
: replyTokens(JSON.stringify(lean && !('predicate' in item) ? leanMemory(item) : item)) + 1 + (brings.get(bringsKey(item)) ?? 0);
|
|
1594
|
+
costs.set(item, size);
|
|
1595
|
+
}
|
|
1596
|
+
return size;
|
|
1597
|
+
};
|
|
1598
|
+
// With the lane armed, what the session expansion added (reason SESSION_EXPAND_REASON) is not budgeted with the items.
|
|
1599
|
+
// The lane lists the whole reply when it fits, and its contradictions take the room the items leave; counted as items,
|
|
1600
|
+
// the statements made too big a whole reply that fit without them (the cut that followed listed one fact of three), and
|
|
1601
|
+
// took a contradiction's room. So the lane budgets what it budgets with the expansion off, and the statements take the
|
|
1602
|
+
// room its whole reply leaves (`withLane`).
|
|
1603
|
+
const statement = (memory) => honestyLane && candidateMemories[memory].reason === orchestrator_1.SESSION_EXPAND_REASON;
|
|
1604
|
+
const itemMemories = candidateMemories.filter((_, i) => !statement(i));
|
|
1605
|
+
const itemEvents = memoryEvents.filter((_, i) => !statement(i));
|
|
1606
|
+
const statements = candidateMemories.filter((_, i) => statement(i)).map(memoryWithProvenance);
|
|
1607
|
+
// The facts are cut from their end and the matches by score, but the match the verdict reports takes its room first when
|
|
1608
|
+
// that would not list it, and so does the newest match above the relevance bar (`recent`) when it fits.
|
|
1609
|
+
const budget = (allowance) => {
|
|
1610
|
+
const place = (keep) => pinned(applyBudget(candidateFacts, factEvents, itemMemories, itemEvents, allowance, cost, { keep, closest: missed, recent, mates, sessionStatements, relevance: item => ('text' in item ? statementRelevance(item.text, query) : 0) }), allowance);
|
|
1611
|
+
const plain = place();
|
|
1612
|
+
return match !== undefined && !holds(plain) ? place(match) : plain;
|
|
1613
|
+
};
|
|
1614
|
+
// A replaced memory the budget kept has its correction kept with it: the correction is added when the allowance holds it, and
|
|
1615
|
+
// when it does not the replaced memory is the one left out (the reply says it cut something).
|
|
1616
|
+
const pinned = (kept, allowance) => {
|
|
1617
|
+
if (correctionOf.size === 0)
|
|
1618
|
+
return kept;
|
|
1619
|
+
let memories = kept.memories;
|
|
1620
|
+
let tokens = kept.tokens;
|
|
1621
|
+
let truncated = kept.truncated;
|
|
1622
|
+
for (const memory of kept.memories) {
|
|
1623
|
+
const correction = correctionOf.get(memory);
|
|
1624
|
+
if (correction === undefined || memories.includes(correction))
|
|
1625
|
+
continue;
|
|
1626
|
+
if (tokens + cost(correction) <= allowance) {
|
|
1627
|
+
memories = [...memories, correction];
|
|
1628
|
+
tokens += cost(correction);
|
|
1629
|
+
}
|
|
1630
|
+
else {
|
|
1631
|
+
memories = memories.filter(other => other !== memory);
|
|
1632
|
+
tokens -= cost(memory);
|
|
1633
|
+
truncated = true;
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
if (memories === kept.memories)
|
|
1637
|
+
return kept;
|
|
1638
|
+
return { ...kept, memories: itemMemories.filter(memory => memories.includes(memory)), tokens, truncated };
|
|
1639
|
+
};
|
|
1640
|
+
// What the read-honesty lane adds in the room the items leave, measured whole, so it never costs the reply an item: its
|
|
1641
|
+
// contradictions that no listed fact brings (inside one scope, about a subject the query names), at their least and in
|
|
1642
|
+
// their order while the reply holds them, cut from their end like any list; then more sides of each contradiction
|
|
1643
|
+
// across scopes, first contradiction first, which `total` counts while they are not named (readHonesty.ts
|
|
1644
|
+
// fitConflicts). A contradiction the reply cannot hold is not named, and the reply says it was cut.
|
|
1645
|
+
const withLane = (kept, body) => {
|
|
1646
|
+
const fits = (reply) => replyTokens(JSON.stringify(reply)) <= maxTokens;
|
|
1647
|
+
const found = contradictions(kept.facts);
|
|
1648
|
+
const least = (0, readHonesty_1.fitConflicts)(found, 0);
|
|
1649
|
+
const take = found.map(c => kept.facts.some(f => about(f, c))); // in the reply already: their facts paid for them
|
|
1650
|
+
for (let i = 0; i < found.length; i++) {
|
|
1651
|
+
if (take[i])
|
|
1652
|
+
continue;
|
|
1653
|
+
take[i] = true;
|
|
1654
|
+
if (!fits({ ...body, conflicts: least.filter((_, j) => take[j]) })) {
|
|
1655
|
+
take[i] = false;
|
|
1656
|
+
break;
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1659
|
+
const shown = (0, readHonesty_1.fitConflicts)(found.filter((_, i) => take[i]), conflicts => fits({ ...body, conflicts }));
|
|
1660
|
+
const reply = { ...body, conflicts: shown };
|
|
1661
|
+
if (shown.length < found.length)
|
|
1662
|
+
reply.truncated = true;
|
|
1663
|
+
// Then what the session expansion added, after every match in the order it was laid out, while the whole reply holds
|
|
1664
|
+
// it: the room the reply leaves with the expansion off. As in the default lane (applyBudget), `truncated` does not
|
|
1665
|
+
// count it.
|
|
1666
|
+
for (const memory of statements) {
|
|
1667
|
+
const memories = [...reply.memories, memory];
|
|
1668
|
+
if (!fits({ ...reply, memories }))
|
|
1669
|
+
break;
|
|
1670
|
+
reply.memories = memories;
|
|
1671
|
+
}
|
|
1672
|
+
if (kept.facts.length > 0 || kept.memories.length > 0)
|
|
1673
|
+
return reply;
|
|
1674
|
+
// A reply that lists no item can be over all the same: its fixed part alone, for a read with nothing on file or a
|
|
1675
|
+
// budget too small for any item. It drops what the lane adds, the least of it first, until it fits: the provenance
|
|
1676
|
+
// legend, which explains items it does not list; how the abstention was reached (the terms it looked for, then its
|
|
1677
|
+
// coverage, bar and count); a list of contradictions that names none; and last the abstention itself, which a budget
|
|
1678
|
+
// that cannot hold it does not carry, as it does not carry a contradiction, and the reply says it was cut. What is
|
|
1679
|
+
// left is the reply with the lane off, over only where that one is (an echoed query longer than the budget).
|
|
1680
|
+
const abstention = reply.abstention;
|
|
1681
|
+
const cuts = [
|
|
1682
|
+
() => { delete reply.provenance; },
|
|
1683
|
+
() => {
|
|
1684
|
+
if (!abstention)
|
|
1685
|
+
return;
|
|
1686
|
+
const { terms: _terms, ...reached } = abstention;
|
|
1687
|
+
reply.abstention = reached;
|
|
1688
|
+
},
|
|
1689
|
+
() => { if (abstention)
|
|
1690
|
+
reply.abstention = { status: abstention.status, reason: abstention.reason, statement: abstention.statement }; },
|
|
1691
|
+
() => { if (shown.length === 0)
|
|
1692
|
+
delete reply.conflicts; },
|
|
1693
|
+
() => {
|
|
1694
|
+
delete reply.abstention;
|
|
1695
|
+
if (abstention)
|
|
1696
|
+
reply.truncated = true;
|
|
1697
|
+
},
|
|
1698
|
+
];
|
|
1699
|
+
for (const cut of cuts) {
|
|
1700
|
+
if (fits(reply))
|
|
1701
|
+
break;
|
|
1702
|
+
cut();
|
|
1703
|
+
}
|
|
1704
|
+
return reply;
|
|
1705
|
+
};
|
|
1706
|
+
const fixed = () => budget(maxTokens - listedTokens(assemble({ facts: [], memories: [], truncated: false, tokens: 0, listed: new Set() }, judged?.relevance)));
|
|
1707
|
+
// With the lane armed, the reply that holds everything the read found is the reply whenever it fits: the budget cuts
|
|
1708
|
+
// each list in turn, so short of that it can keep a fact's ledger event where the fact would take its place.
|
|
1709
|
+
const whole = honestyLane ? budget(Number.POSITIVE_INFINITY) : undefined;
|
|
1710
|
+
let kept = whole ?? fixed();
|
|
1711
|
+
let body = assemble(kept);
|
|
1712
|
+
if (whole && replyTokens(JSON.stringify(body)) > maxTokens) {
|
|
1713
|
+
kept = fixed();
|
|
1714
|
+
body = assemble(kept);
|
|
1715
|
+
}
|
|
1716
|
+
for (let over = listedTokens(body) - maxTokens; over > 0 && kept.tokens > 0; over = listedTokens(body) - maxTokens) {
|
|
1717
|
+
kept = budget(kept.tokens - over);
|
|
1718
|
+
body = assemble(kept);
|
|
1719
|
+
}
|
|
1720
|
+
const reply = honestyLane ? withLane(kept, body) : body;
|
|
1721
|
+
if (rankedBy === 'session_date') {
|
|
1722
|
+
// Selection is finished, including the honesty lane's extra statements. Render those statements with their
|
|
1723
|
+
// session: appending a newest update after an older match contradicts the order the reply promises. Summary
|
|
1724
|
+
// tables stay outside ranking, and word-only matches follow the dated lane in their original order.
|
|
1725
|
+
const listed = reply.memories;
|
|
1726
|
+
const leading = (0, digests_1.leadingSummaryHits)(listed);
|
|
1727
|
+
const dated = [];
|
|
1728
|
+
const keywords = [];
|
|
1729
|
+
for (const memory of listed.slice(leading)) {
|
|
1730
|
+
(memory.reason.split('+').every(r => WORD_MATCH_REASONS.includes(r)) ? keywords : dated).push(memory);
|
|
1731
|
+
}
|
|
1732
|
+
reply.memories = [
|
|
1733
|
+
...listed.slice(0, leading),
|
|
1734
|
+
...(0, orchestrator_1.orderHits)(dated.map(item => ({ item, score: item.score ?? 0, session_date: item.session_date })), 'session_date').map(hit => hit.item),
|
|
1735
|
+
...keywords,
|
|
1736
|
+
];
|
|
1737
|
+
}
|
|
1738
|
+
if (correctionOf.size > 0)
|
|
1739
|
+
pinToCorrections(reply, correctionOf);
|
|
1740
|
+
// Last, once the order and the pairs are settled on the full items, which say how they were found.
|
|
1741
|
+
if (lean)
|
|
1742
|
+
reply.memories = reply.memories.map(leanMemory);
|
|
1743
|
+
return reply;
|
|
1744
|
+
}
|