renor-engine 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +13 -8
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/api/server.js +10 -8
  6. package/dist/buildIdentity.js +18 -1
  7. package/dist/callPhases.js +82 -0
  8. package/dist/cli.js +36 -1
  9. package/dist/context/rowStore.js +1 -1
  10. package/dist/conversation/parser.js +40 -36
  11. package/dist/db/client.js +6 -4
  12. package/dist/db/handoffTable.js +31 -0
  13. package/dist/db/indexSalvage.js +24 -4
  14. package/dist/db/migrations.js +30 -2
  15. package/dist/db/sqlite.js +396 -0
  16. package/dist/digests/digestCache.js +6 -2
  17. package/dist/digests/digests.js +3 -3
  18. package/dist/digests/worker.js +3 -1
  19. package/dist/embeddings/caching.js +5 -5
  20. package/dist/embeddings/local.js +5 -3
  21. package/dist/embeddings/openai.js +4 -3
  22. package/dist/embeddings/simdScan.js +3 -1
  23. package/dist/embeddings/vectorIndex.js +0 -1
  24. package/dist/embeddings/vertex.js +123 -13
  25. package/dist/eval/cacheReport.js +7 -0
  26. package/dist/eval/context700-candidate.js +5 -5
  27. package/dist/eval/evalCache.js +46 -3
  28. package/dist/eval/indexSnapshots.js +549 -0
  29. package/dist/eval/pageDump.js +67 -0
  30. package/dist/eval/runManifest.js +6 -0
  31. package/dist/eval/textTokens.js +17 -0
  32. package/dist/evidence/attach.js +0 -3
  33. package/dist/evidence/evidenceCache.js +3 -3
  34. package/dist/evidence/ollamaChat.js +1 -1
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/jevDecisionModel.js +307 -0
  39. package/dist/extraction/preferenceExtractor.js +1 -4
  40. package/dist/extraction/secretGuard.js +7 -3
  41. package/dist/extraction/typedFacts.js +1 -1
  42. package/dist/importing/apply.js +42 -14
  43. package/dist/importing/cli.js +9 -8
  44. package/dist/importing/existing.js +2 -0
  45. package/dist/importing/facts.js +0 -2
  46. package/dist/importing/plan.js +0 -5
  47. package/dist/importing/source.js +1 -1
  48. package/dist/indexer/parser.js +4 -1
  49. package/dist/indexer/pipeline.js +99 -279
  50. package/dist/indexer/walker.js +11 -25
  51. package/dist/learning/cli.js +113 -0
  52. package/dist/learning/schema.js +144 -0
  53. package/dist/learning/settings.js +45 -0
  54. package/dist/learning/store.js +267 -0
  55. package/dist/learning/tap.js +401 -0
  56. package/dist/mcp/server.js +36 -6
  57. package/dist/memory/dreamer/adjudicate.js +215 -0
  58. package/dist/memory/dreamer/apply.js +120 -0
  59. package/dist/memory/dreamer/budget.js +64 -0
  60. package/dist/memory/dreamer/candidates.js +149 -0
  61. package/dist/memory/dreamer/collection.js +104 -0
  62. package/dist/memory/dreamer/dreamer.js +352 -0
  63. package/dist/memory/dreamer/review.js +111 -0
  64. package/dist/memory/factConflicts.js +2 -0
  65. package/dist/memory/handoff.js +363 -0
  66. package/dist/memory/lifecycle.js +338 -80
  67. package/dist/memory/sourceName.js +77 -1
  68. package/dist/memory/statusFacts.js +61 -0
  69. package/dist/memory/writer.js +3 -2
  70. package/dist/memorymap/memoryMap.js +39 -13
  71. package/dist/memorymap/replacements.js +8 -3
  72. package/dist/mvp/agentMemory.js +109 -50
  73. package/dist/mvp/answerCheck.js +211 -0
  74. package/dist/mvp/cli.js +14 -1
  75. package/dist/mvp/compileCache.js +3 -10
  76. package/dist/mvp/compileSeed.js +9 -3
  77. package/dist/mvp/config.js +4 -0
  78. package/dist/mvp/contextDistinct.js +53 -0
  79. package/dist/mvp/contextProfile.js +87 -0
  80. package/dist/mvp/directorySurface.js +68 -0
  81. package/dist/mvp/engine.js +39 -26
  82. package/dist/mvp/exportAll.js +36 -2
  83. package/dist/mvp/factMarkers.js +18 -2
  84. package/dist/mvp/history.js +5 -4
  85. package/dist/mvp/log.js +118 -0
  86. package/dist/mvp/server.js +759 -136
  87. package/dist/mvp/serverCard.js +12 -3
  88. package/dist/mvp/telemetry.js +35 -2
  89. package/dist/net/poolFetch.js +174 -0
  90. package/dist/recall/actionRecall.js +282 -0
  91. package/dist/recall/autoRecall.js +447 -0
  92. package/dist/recall/searchRecall.js +189 -0
  93. package/dist/retrieval/entityLookup.js +53 -4
  94. package/dist/retrieval/orchestrator.js +148 -10
  95. package/dist/retrieval/plainWording.js +236 -0
  96. package/dist/retrieval/rankingTrace.js +184 -0
  97. package/dist/retrieval/readerLayout.js +143 -224
  98. package/dist/router/routerCache.js +2 -2
  99. package/dist/sharing/cli.js +181 -0
  100. package/dist/sharing/examples.js +136 -0
  101. package/dist/sharing/fields.js +225 -0
  102. package/dist/sharing/loop.js +80 -0
  103. package/dist/sharing/project.js +327 -0
  104. package/dist/sharing/send.js +466 -0
  105. package/dist/sharing/state.js +222 -0
  106. package/dist/sharing/wording.js +48 -0
  107. package/dist/watcher/watcher.js +5 -3
  108. package/package.json +5 -7
  109. package/dist/BUILD.json +0 -6
  110. package/dist/importing/index.js +0 -29
@@ -0,0 +1,327 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.LABEL_WINDOW_MS = void 0;
4
+ exports.hourOf = hourOf;
5
+ exports.projectRows = projectRows;
6
+ const fields_1 = require("./fields");
7
+ /**
8
+ * The join, done inside the user's own folder: the learning log says what was served, saved, cited and retired; this turns
9
+ * it into finished, labelled rows. Only these rows can ever leave. They carry no key, no memory id, no query and no text
10
+ * (text only at `data`, from the example store), and every time is coarsened to the hour.
11
+ *
12
+ * A row is sent only once its label is final: the window (default 24 hours) after the event has closed. Until then the event
13
+ * waits, and the sender's watermark says how far it has got.
14
+ */
15
+ exports.LABEL_WINDOW_MS = 24 * 3_600_000;
16
+ const asString = (value) => (typeof value === 'string' ? value : null);
17
+ const asNumber = (value) => (typeof value === 'number' && Number.isFinite(value) ? value : null);
18
+ const asBool = (value) => (value === 1 || value === true ? true : value === 0 || value === false ? false : null);
19
+ /** `2026-10-21T14`: the hour a row happened in, in UTC. */
20
+ function hourOf(ms) {
21
+ return new Date(ms).toISOString().slice(0, 13);
22
+ }
23
+ function delayBucket(ms) {
24
+ if (ms < 60_000)
25
+ return '<1m';
26
+ if (ms < 600_000)
27
+ return '<10m';
28
+ if (ms < 3_600_000)
29
+ return '<1h';
30
+ if (ms < 86_400_000)
31
+ return '<1d';
32
+ return '>=1d';
33
+ }
34
+ function lengthBucket(ms) {
35
+ if (ms < 60_000)
36
+ return '<1m';
37
+ if (ms < 600_000)
38
+ return '<10m';
39
+ if (ms < 3_600_000)
40
+ return '<1h';
41
+ if (ms < 21_600_000)
42
+ return '<6h';
43
+ return '>=6h';
44
+ }
45
+ function countBucket(n) {
46
+ if (n <= 0)
47
+ return '0';
48
+ if (n === 1)
49
+ return '1';
50
+ if (n <= 3)
51
+ return '2-3';
52
+ if (n <= 7)
53
+ return '4-7';
54
+ return '8+';
55
+ }
56
+ /** The smallest power of two at or above `tokens`: a size class, not a size. */
57
+ function tokenBucket(tokens) {
58
+ if (tokens === null || tokens < 0)
59
+ return null;
60
+ let bucket = 1;
61
+ while (bucket < tokens)
62
+ bucket *= 2;
63
+ return bucket;
64
+ }
65
+ const round = (value) => (value === null ? null : Math.round(value * 10_000) / 10_000);
66
+ /** Events about a memory, by its key, in time order. */
67
+ function indexEvents(tables) {
68
+ const byMemory = new Map();
69
+ const bySubject = new Map();
70
+ const add = (map, key, event) => {
71
+ if (key === null)
72
+ return;
73
+ const list = map.get(key);
74
+ if (list)
75
+ list.push(event);
76
+ else
77
+ map.set(key, [event]);
78
+ };
79
+ for (const op of tables.memory_ops) {
80
+ const at = asNumber(op.at);
81
+ const kind = op.op;
82
+ if (at === null)
83
+ continue;
84
+ const key = asString(op.memory_key);
85
+ if (kind === 'retrieve')
86
+ add(byMemory, key, { at, kind: 'retrieve' });
87
+ else if (kind === 'delete')
88
+ add(byMemory, key, { at, kind: 'delete' });
89
+ else if (kind === 'supersede') {
90
+ // A supersede changes the memory it replaces; it is also the save of the new value (its own key).
91
+ add(byMemory, asString(op.replaced_key), { at, kind: 'supersede' });
92
+ if (asString(op.replaced_key) !== key)
93
+ add(byMemory, key, { at, kind: 'supersede' });
94
+ }
95
+ }
96
+ for (const row of tables.feedback) {
97
+ const at = asNumber(row.at);
98
+ if (at === null)
99
+ continue;
100
+ add(byMemory, asString(row.memory_key), { at, kind: 'cite' });
101
+ add(bySubject, asString(row.subject_key), { at, kind: 'cite' });
102
+ }
103
+ for (const list of [...byMemory.values(), ...bySubject.values()])
104
+ list.sort((a, b) => a.at - b.at);
105
+ return { byMemory, bySubject };
106
+ }
107
+ function labelOf(lists, from, until, skipSupersedeAt = null) {
108
+ const label = { opened: false, cited: false, updated: false, retired: false, firstAt: null };
109
+ for (const list of lists) {
110
+ for (const event of list ?? []) {
111
+ if (event.at <= from || event.at > until)
112
+ continue;
113
+ if (skipSupersedeAt !== null && event.kind === 'supersede' && event.at === skipSupersedeAt)
114
+ continue;
115
+ if (event.kind === 'retrieve')
116
+ label.opened = true;
117
+ else if (event.kind === 'cite')
118
+ label.cited = true;
119
+ else if (event.kind === 'supersede')
120
+ label.updated = true;
121
+ else
122
+ label.retired = true;
123
+ label.firstAt = label.firstAt === null ? event.at : Math.min(label.firstAt, event.at);
124
+ }
125
+ }
126
+ return label;
127
+ }
128
+ /** A candidate the reply served (rank set). The log also keeps the ones the ranker scored and the reply did not serve: never shared. */
129
+ function servedRow(candidate) {
130
+ return asNumber(candidate.rank) !== null;
131
+ }
132
+ function retrievalRows(log, options, windowMs) {
133
+ const { tables } = log;
134
+ const events = indexEvents(tables);
135
+ const cutoff = options.now - windowMs;
136
+ const candidatesBy = new Map();
137
+ for (const candidate of tables.candidates) {
138
+ const id = asNumber(candidate.retrieval_id);
139
+ if (id === null || !servedRow(candidate))
140
+ continue;
141
+ const list = candidatesBy.get(id);
142
+ if (list)
143
+ list.push(candidate);
144
+ else
145
+ candidatesBy.set(id, [candidate]);
146
+ }
147
+ const rows = [];
148
+ const retrievals = [...tables.retrievals].sort((a, b) => (asNumber(a.at) ?? 0) - (asNumber(b.at) ?? 0) || (asNumber(a.id) ?? 0) - (asNumber(b.id) ?? 0));
149
+ for (const retrieval of retrievals) {
150
+ const at = asNumber(retrieval.at);
151
+ const id = asNumber(retrieval.id);
152
+ const tool = asString(retrieval.tool);
153
+ if (at === null || id === null || tool === null || !fields_1.RETRIEVAL_TOOL_NAMES.includes(tool))
154
+ continue;
155
+ if (at <= options.watermarks.retrieval || at > cutoff)
156
+ continue;
157
+ const queryKey = asString(retrieval.query_key);
158
+ const candidates = (candidatesBy.get(id) ?? []).slice(0, 200).map(candidate => {
159
+ const kind = asString(candidate.kind) ?? 'unkeyed';
160
+ const memoryKey = asString(candidate.memory_key);
161
+ const joinable = memoryKey !== null && kind !== 'unkeyed';
162
+ const label = joinable
163
+ ? labelOf([events.byMemory.get(memoryKey), kind === 'fact' ? events.bySubject.get(asString(candidate.subject_key) ?? '') : undefined], at, at + windowMs)
164
+ : null;
165
+ const out = {
166
+ rank: asNumber(candidate.rank) ?? 0,
167
+ kind,
168
+ lane: asString(candidate.reason),
169
+ score: round(asNumber(candidate.score)),
170
+ matched: asBool(candidate.matched),
171
+ current: asBool(candidate.current),
172
+ opened: label === null ? null : label.opened,
173
+ cited: label === null ? null : label.cited,
174
+ updated: label === null ? null : label.updated,
175
+ retired: label === null ? null : label.retired,
176
+ delay_bucket: label === null || label.firstAt === null ? null : delayBucket(label.firstAt - at),
177
+ };
178
+ if (options.level === 'data')
179
+ out.text = memoryKey === null ? null : options.examples?.memory(memoryKey) ?? null;
180
+ return out;
181
+ });
182
+ const row = {
183
+ kind: 'retrieval-labelled/1',
184
+ hour: hourOf(at),
185
+ tool,
186
+ ranker: asString(retrieval.ranker) ?? 'baseline',
187
+ build: asString(retrieval.build),
188
+ model: (0, fields_1.modelIdentifier)(asString(retrieval.model)),
189
+ retrieval: asString(retrieval.retrieval),
190
+ served: asNumber(retrieval.served) ?? candidates.length,
191
+ reply_tokens_bucket: tokenBucket(asNumber(retrieval.reply_tokens)),
192
+ relevant: asBool(retrieval.relevant),
193
+ truncated: asBool(retrieval.truncated),
194
+ candidates,
195
+ };
196
+ if (options.level === 'data')
197
+ row.query = queryKey === null ? null : options.examples?.query(queryKey) ?? null;
198
+ rows.push({ kind: 'retrieval', at, row });
199
+ }
200
+ return rows;
201
+ }
202
+ function saveRows(log, options, windowMs) {
203
+ const { tables } = log;
204
+ const events = indexEvents(tables);
205
+ const cutoff = options.now - windowMs;
206
+ // How many times each memory was served after a given time: candidates by key, with the retrieval's time.
207
+ const retrievalAt = new Map();
208
+ for (const retrieval of tables.retrievals) {
209
+ const id = asNumber(retrieval.id);
210
+ const at = asNumber(retrieval.at);
211
+ if (id !== null && at !== null)
212
+ retrievalAt.set(id, at);
213
+ }
214
+ const servedBy = new Map();
215
+ for (const candidate of tables.candidates) {
216
+ const key = asString(candidate.memory_key);
217
+ const id = asNumber(candidate.retrieval_id);
218
+ const at = id === null ? undefined : retrievalAt.get(id);
219
+ if (key === null || at === undefined || !servedRow(candidate))
220
+ continue;
221
+ const list = servedBy.get(key);
222
+ if (list)
223
+ list.push(at);
224
+ else
225
+ servedBy.set(key, [at]);
226
+ }
227
+ const rows = [];
228
+ const ops = [...tables.memory_ops].sort((a, b) => (asNumber(a.at) ?? 0) - (asNumber(b.at) ?? 0) || (asNumber(a.id) ?? 0) - (asNumber(b.id) ?? 0));
229
+ for (const op of ops) {
230
+ const at = asNumber(op.at);
231
+ const tool = asString(op.tool);
232
+ const kind = asString(op.op);
233
+ if (at === null || tool === null || !fields_1.SAVE_TOOL_NAMES.includes(tool))
234
+ continue;
235
+ if (kind !== 'create' && kind !== 'supersede')
236
+ continue;
237
+ if (at <= options.watermarks.save || at > cutoff)
238
+ continue;
239
+ const memoryKey = asString(op.memory_key);
240
+ const item = asString(op.kind) ?? 'unkeyed';
241
+ const joinable = memoryKey !== null && item !== 'unkeyed';
242
+ const label = joinable
243
+ ? labelOf([events.byMemory.get(memoryKey), item === 'fact' ? events.bySubject.get(asString(op.subject_key) ?? '') : undefined], at, at + windowMs, at)
244
+ : labelOf([], at, at + windowMs);
245
+ const reads = joinable ? (servedBy.get(memoryKey) ?? []).filter(time => time > at && time <= at + windowMs).length : 0;
246
+ const hasCites = tables.feedback.some(cite => cite.tool === tool && Math.abs((asNumber(cite.at) ?? Number.NaN) - at) <= 2000
247
+ && cite.agent_key === op.agent_key && cite.session_key === op.session_key);
248
+ const row = {
249
+ kind: 'save-outcome/1',
250
+ hour: hourOf(at),
251
+ tool,
252
+ op: kind,
253
+ item,
254
+ has_cites: hasCites,
255
+ reads_bucket: countBucket(reads),
256
+ opened: label.opened,
257
+ cited: label.cited,
258
+ superseded: label.updated,
259
+ retired: label.retired,
260
+ };
261
+ if (options.level === 'data')
262
+ row.text = memoryKey === null ? null : options.examples?.memory(memoryKey) ?? null;
263
+ rows.push({ kind: 'save', at, row });
264
+ }
265
+ return rows;
266
+ }
267
+ function sessionRows(log, options, windowMs) {
268
+ const { tables } = log;
269
+ const cutoff = options.now - windowMs;
270
+ const sessions = new Map();
271
+ const touch = (key, at, tool, apply) => {
272
+ if (key === null || at === null || tool === null)
273
+ return;
274
+ let seen = sessions.get(key);
275
+ if (!seen) {
276
+ seen = { first: at, last: at, tool, reads: 0, saves: 0, empty: 0 };
277
+ sessions.set(key, seen);
278
+ }
279
+ if (at < seen.first) {
280
+ seen.first = at;
281
+ seen.tool = tool;
282
+ }
283
+ seen.last = Math.max(seen.last, at);
284
+ apply(seen);
285
+ };
286
+ for (const retrieval of tables.retrievals) {
287
+ touch(asString(retrieval.session_key), asNumber(retrieval.at), asString(retrieval.tool), seen => {
288
+ seen.reads += 1;
289
+ if ((asNumber(retrieval.served) ?? 0) === 0)
290
+ seen.empty += 1;
291
+ });
292
+ }
293
+ for (const op of tables.memory_ops) {
294
+ const tool = asString(op.tool);
295
+ touch(asString(op.session_key), asNumber(op.at), tool, seen => {
296
+ if (tool !== null && fields_1.SAVE_TOOL_NAMES.includes(tool) && (op.op === 'create' || op.op === 'supersede'))
297
+ seen.saves += 1;
298
+ });
299
+ }
300
+ const rows = [];
301
+ for (const seen of [...sessions.values()].sort((a, b) => a.last - b.last)) {
302
+ if (seen.last <= options.watermarks.session || seen.last > cutoff)
303
+ continue;
304
+ const firstCall = fields_1.RETRIEVAL_TOOL_NAMES.includes(seen.tool) || fields_1.SAVE_TOOL_NAMES.includes(seen.tool)
305
+ ? seen.tool
306
+ : 'other';
307
+ rows.push({
308
+ kind: 'session',
309
+ at: seen.last,
310
+ row: {
311
+ kind: 'session-aggregate/1',
312
+ hour: hourOf(seen.first),
313
+ length_bucket: lengthBucket(seen.last - seen.first),
314
+ reads_bucket: countBucket(seen.reads),
315
+ saves_bucket: countBucket(seen.saves),
316
+ empty_reads_bucket: countBucket(seen.empty),
317
+ first_call: firstCall,
318
+ },
319
+ });
320
+ }
321
+ return rows;
322
+ }
323
+ /** The finished rows the log holds that are final and not yet sent, oldest first within each kind. */
324
+ function projectRows(log, options) {
325
+ const windowMs = options.windowMs ?? exports.LABEL_WINDOW_MS;
326
+ return [...retrievalRows(log, options, windowMs), ...saveRows(log, options, windowMs), ...sessionRows(log, options, windowMs)];
327
+ }