renor-engine 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +7 -2
  2. package/dist/.tsbuildinfo +1 -1
  3. package/dist/cli.js +26 -0
  4. package/dist/eval/indexSnapshots.js +1 -0
  5. package/dist/evidence/ollamaChat.js +1 -1
  6. package/dist/extraction/jevDecisionModel.js +307 -0
  7. package/dist/extraction/secretGuard.js +7 -3
  8. package/dist/learning/cli.js +113 -0
  9. package/dist/learning/schema.js +144 -0
  10. package/dist/learning/settings.js +45 -0
  11. package/dist/learning/store.js +267 -0
  12. package/dist/learning/tap.js +401 -0
  13. package/dist/memory/dreamer/adjudicate.js +215 -0
  14. package/dist/memory/dreamer/apply.js +120 -0
  15. package/dist/memory/dreamer/budget.js +64 -0
  16. package/dist/memory/dreamer/candidates.js +149 -0
  17. package/dist/memory/dreamer/collection.js +104 -0
  18. package/dist/memory/dreamer/dreamer.js +352 -0
  19. package/dist/memory/dreamer/review.js +111 -0
  20. package/dist/memory/factConflicts.js +2 -0
  21. package/dist/memory/lifecycle.js +28 -5
  22. package/dist/memory/writer.js +3 -2
  23. package/dist/mvp/agentMemory.js +3 -0
  24. package/dist/mvp/cli.js +11 -0
  25. package/dist/mvp/config.js +4 -0
  26. package/dist/mvp/exportAll.js +19 -0
  27. package/dist/mvp/server.js +118 -10
  28. package/dist/retrieval/orchestrator.js +69 -16
  29. package/dist/retrieval/rankingTrace.js +184 -0
  30. package/dist/sharing/cli.js +181 -0
  31. package/dist/sharing/examples.js +136 -0
  32. package/dist/sharing/fields.js +225 -0
  33. package/dist/sharing/loop.js +80 -0
  34. package/dist/sharing/project.js +327 -0
  35. package/dist/sharing/send.js +466 -0
  36. package/dist/sharing/state.js +222 -0
  37. package/dist/sharing/wording.js +48 -0
  38. package/package.json +1 -1
  39. package/dist/BUILD.json +0 -6
@@ -0,0 +1,45 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.MAX_LEARNING_RETENTION_DAYS = exports.DEFAULT_LEARNING_RETENTION_DAYS = void 0;
7
+ exports.learningDbPath = learningDbPath;
8
+ exports.learningRetentionDays = learningRetentionDays;
9
+ exports.learningSettings = learningSettings;
10
+ const node_path_1 = __importDefault(require("node:path"));
11
+ /**
12
+ * Learning-log settings, read from the environment. Off by default: with nothing set, no file is made and no call is recorded.
13
+ * A bad value is an error at start-up, as RENOR_ON_DEMAND is, so a typo can never leave the log on or off by accident.
14
+ */
15
+ exports.DEFAULT_LEARNING_RETENTION_DAYS = 30;
16
+ exports.MAX_LEARNING_RETENTION_DAYS = 365;
17
+ /** The learning log's file for one memory folder. Each tenant has its own memory folder, so each tenant has its own log. */
18
+ function learningDbPath(memoryDir) {
19
+ return node_path_1.default.join(node_path_1.default.resolve(memoryDir), '.memory-index', 'learning.sqlite');
20
+ }
21
+ function onOff(name, raw) {
22
+ const value = (raw ?? '').trim().toLowerCase();
23
+ if (value === '' || value === '0' || value === 'false')
24
+ return false;
25
+ if (value === '1' || value === 'true')
26
+ return true;
27
+ throw new Error(`${name} must be 1/true or 0/false`);
28
+ }
29
+ /** RENOR_LEARNING_RETENTION_DAYS, validated. The management commands read it too, whether or not the log is on. */
30
+ function learningRetentionDays(env = process.env) {
31
+ const raw = (env.RENOR_LEARNING_RETENTION_DAYS ?? '').trim();
32
+ const days = raw === '' ? exports.DEFAULT_LEARNING_RETENTION_DAYS : Number(raw);
33
+ if (!Number.isInteger(days) || days < 1 || days > exports.MAX_LEARNING_RETENTION_DAYS) {
34
+ throw new Error(`RENOR_LEARNING_RETENTION_DAYS must be an integer between 1 and ${exports.MAX_LEARNING_RETENTION_DAYS}`);
35
+ }
36
+ return days;
37
+ }
38
+ function learningSettings(memoryDir, env = process.env) {
39
+ const enabled = onOff('RENOR_LEARNING_LOG', env.RENOR_LEARNING_LOG);
40
+ const rawQueries = onOff('RENOR_LEARNING_RAW_QUERY', env.RENOR_LEARNING_RAW_QUERY);
41
+ if (rawQueries && !enabled) {
42
+ throw new Error('RENOR_LEARNING_RAW_QUERY needs RENOR_LEARNING_LOG=1: a query is only ever kept by a log that is on');
43
+ }
44
+ return { enabled, retentionDays: learningRetentionDays(env), rawQueries, dbPath: learningDbPath(memoryDir) };
45
+ }
@@ -0,0 +1,267 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.LearningLog = void 0;
7
+ exports.learningLogAt = learningLogAt;
8
+ exports.closeLearningLogs = closeLearningLogs;
9
+ const sqlite_1 = __importDefault(require("../db/sqlite"));
10
+ const node_crypto_1 = require("node:crypto");
11
+ const node_fs_1 = __importDefault(require("node:fs"));
12
+ const node_path_1 = __importDefault(require("node:path"));
13
+ const secretScanner_1 = require("../extraction/secretScanner");
14
+ const schema_1 = require("./schema");
15
+ /**
16
+ * The learning log: one SQLite file per memory folder, written by the MCP tools when RENOR_LEARNING_LOG is on (tap.ts) and
17
+ * read by the learner through `exportLog()`. It is separate from product analytics (telemetry.ts) and from the index, so
18
+ * deleting the index never deletes it and deleting the log never touches memory.
19
+ *
20
+ * The secret that makes the keys is written once, in the `meta` table, when the log is created; `exportLog()` never includes
21
+ * it, and `deleteAll()` replaces it, so keys made before a delete cannot be matched to keys made after one.
22
+ */
23
+ const DAY_MS = 86_400_000;
24
+ const PRUNE_EVERY_MS = 3_600_000;
25
+ /** The order an export lists each table in: ids where a table has one, otherwise its key. */
26
+ const ORDER = {
27
+ retrievals: 'id',
28
+ candidates: 'retrieval_id, id',
29
+ memory_ops: 'id',
30
+ feedback: 'id',
31
+ outcomes: 'id',
32
+ };
33
+ const flag = (value) => (value === null ? null : value ? 1 : 0);
34
+ const OUTCOME_OBSERVED = { verifier: 1, user: 1, proxy: 0 };
35
+ class LearningLog {
36
+ file;
37
+ db;
38
+ retentionDays;
39
+ rawQueries;
40
+ now;
41
+ secret;
42
+ lastPrune;
43
+ constructor(file, db, secret, retentionDays, rawQueries, now) {
44
+ this.file = file;
45
+ this.db = db;
46
+ this.retentionDays = retentionDays;
47
+ this.rawQueries = rawQueries;
48
+ this.now = now;
49
+ this.secret = secret;
50
+ // Zero makes the first read or write enforce retention immediately.
51
+ this.lastPrune = 0;
52
+ }
53
+ /** Opens the log at `file`, creating it (owner-only) when `create` is set. Returns null when there is no log and none may be made. */
54
+ static open(file, options) {
55
+ const now = options.now ?? Date.now;
56
+ if (options.create !== true && !node_fs_1.default.existsSync(file))
57
+ return null;
58
+ // Owner-only, like the index (db/client.ts). Tighten an existing file too: it may have been
59
+ // restored with a permissive umask and it contains linkable pseudonymous activity.
60
+ node_fs_1.default.mkdirSync(node_path_1.default.dirname(file), { recursive: true, mode: 0o700 });
61
+ node_fs_1.default.closeSync(node_fs_1.default.openSync(file, node_fs_1.default.constants.O_CREAT | node_fs_1.default.constants.O_WRONLY, 0o600));
62
+ node_fs_1.default.chmodSync(file, 0o600);
63
+ const db = new sqlite_1.default(file);
64
+ try {
65
+ db.pragma('journal_mode = WAL');
66
+ db.pragma('foreign_keys = ON');
67
+ // Deleted rows are overwritten, not left in free pages: a delete means the content is gone from the file.
68
+ db.pragma('secure_delete = ON');
69
+ db.pragma('busy_timeout = 5000');
70
+ db.exec(schema_1.LEARNING_DDL);
71
+ const secret = ensureMeta(db, now());
72
+ return new LearningLog(file, db, secret, options.retentionDays, options.rawQueries, now);
73
+ }
74
+ catch (err) {
75
+ db.close();
76
+ throw err;
77
+ }
78
+ }
79
+ /**
80
+ * A keyed hash of `parts` under this log's secret, 128 bits in hex. `kind` separates the namespaces (memory, query, session...);
81
+ * the parts are JSON-encoded, so ["a b"] and ["a", "b"] cannot collide.
82
+ */
83
+ key(kind, ...parts) {
84
+ return (0, node_crypto_1.createHmac)('sha256', this.secret).update(JSON.stringify([kind, ...parts])).digest('hex').slice(0, 32);
85
+ }
86
+ recordRetrieval(record) {
87
+ const at = record.at ?? this.now();
88
+ // The raw query is written only by a log opened with rawQueries, and never when the secret scanner flags it: the check
89
+ // lives here, at the write, so no caller can keep a query by passing it in.
90
+ // query_redacted means one thing: the scanner flagged a query that the opt-in would have kept.
91
+ const raw = record.queryText;
92
+ const flagged = raw !== null && this.rawQueries && (0, secretScanner_1.scanSecrets)(raw) !== null;
93
+ const keepRaw = raw !== null && this.rawQueries && !flagged;
94
+ const queryRedacted = record.queryRedacted || flagged;
95
+ return this.write(() => {
96
+ const info = this.db.prepare(`INSERT INTO retrievals (at, tool, actor, session_key, agent_key, query_key, query_text, query_redacted, scope_key, retrieval,
97
+ ranker, model, build, served, ms, reply_tokens, relevant, truncated, lean_items)
98
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(at, record.tool, record.actor, record.sessionKey, record.agentKey, record.queryKey, keepRaw ? raw : null, queryRedacted ? 1 : 0, record.scopeKey, record.retrieval, record.ranker, record.model, record.build, record.candidates.filter(candidate => candidate.rank !== null).length, record.ms, record.replyTokens, flag(record.relevant), flag(record.truncated), record.leanItems ? 1 : 0);
99
+ const id = Number(info.lastInsertRowid);
100
+ const columns = [
101
+ 'retrieval_id', 'rank', 'kind', 'memory_key', 'subject_key', 'section_key', 'score', 'base', ...schema_1.TERM_COLUMNS,
102
+ 'pool_rank', 'window_pos', 'budget_kept', 'reason', 'matched', 'current',
103
+ ];
104
+ const insert = this.db.prepare(`INSERT INTO candidates (${columns.join(', ')}) VALUES (${columns.map(() => '?').join(', ')})`);
105
+ for (const candidate of record.candidates) {
106
+ const terms = candidate.terms ?? schema_1.TERM_COLUMNS.map(() => 0);
107
+ insert.run(id, candidate.rank, candidate.kind, candidate.memoryKey, candidate.subjectKey, candidate.sectionKey, candidate.score, candidate.base, ...terms, candidate.poolRank, candidate.windowPos, flag(candidate.budgetKept), candidate.reason, flag(candidate.matched), flag(candidate.current));
108
+ }
109
+ return id;
110
+ });
111
+ }
112
+ recordMemoryOp(record) {
113
+ this.write(() => {
114
+ this.db.prepare(`INSERT INTO memory_ops (at, actor, session_key, agent_key, tool, op, kind, memory_key, subject_key, replaced_key, ms)
115
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(record.at ?? this.now(), record.actor, record.sessionKey, record.agentKey, record.tool, record.op, record.kind, record.memoryKey, record.subjectKey, record.replacedKey, record.ms);
116
+ });
117
+ }
118
+ /** An explicit cite: the caller named this memory as what the save builds on. Always observed: the agent said so. */
119
+ recordFeedback(record) {
120
+ this.write(() => {
121
+ this.db.prepare(`INSERT INTO feedback (at, actor, session_key, agent_key, tool, kind, target, memory_key, subject_key, observed)
122
+ VALUES (?, ?, ?, ?, ?, 'cite', ?, ?, ?, 1)`).run(record.at ?? this.now(), record.actor, record.sessionKey, record.agentKey, record.tool, record.target, record.memoryKey, record.subjectKey);
123
+ });
124
+ }
125
+ /** An outcome of a task. `observed` follows from the basis: a proxy is inferred and is never observed. */
126
+ recordOutcome(record) {
127
+ this.write(() => {
128
+ this.db.prepare(`INSERT INTO outcomes (at, session_key, task_key, outcome, basis, observed) VALUES (?, ?, ?, ?, ?, ?)`).run(record.at ?? this.now(), record.sessionKey, record.taskRef === null ? null : this.key('task', record.taskRef), record.outcome, record.basis, OUTCOME_OBSERVED[record.basis]);
129
+ });
130
+ }
131
+ /** Deletes what is older than the retention window. Runs when the log opens and then hourly while it is written to. */
132
+ prune() {
133
+ const now = this.now();
134
+ this.lastPrune = now;
135
+ const cutoff = now - this.retentionDays * DAY_MS;
136
+ return this.db.transaction(() => {
137
+ const candidates = this.db.prepare('DELETE FROM candidates WHERE retrieval_id IN (SELECT id FROM retrievals WHERE at < ?)').run(cutoff).changes;
138
+ const retrievals = this.db.prepare('DELETE FROM retrievals WHERE at < ?').run(cutoff).changes;
139
+ const memory_ops = this.db.prepare('DELETE FROM memory_ops WHERE at < ?').run(cutoff).changes;
140
+ const feedback = this.db.prepare('DELETE FROM feedback WHERE at < ?').run(cutoff).changes;
141
+ const outcomes = this.db.prepare('DELETE FROM outcomes WHERE at < ?').run(cutoff).changes;
142
+ return { retrievals, candidates, memory_ops, feedback, outcomes };
143
+ })();
144
+ }
145
+ /**
146
+ * Every row inside the retention window, and nothing else: the key secret stays in the file and never leaves it. Rows older
147
+ * than the window are left out even before a prune has deleted them.
148
+ */
149
+ exportLog() {
150
+ this.maybePrune();
151
+ const tables = {};
152
+ const cutoff = this.cutoff();
153
+ for (const table of schema_1.LEARNING_TABLES) {
154
+ tables[table] = table === 'candidates'
155
+ ? this.db.prepare(`SELECT * FROM candidates WHERE retrieval_id IN (SELECT id FROM retrievals WHERE at >= ?) ORDER BY ${ORDER.candidates}`).all(cutoff)
156
+ : this.db.prepare(`SELECT * FROM ${table} WHERE at >= ? ORDER BY ${ORDER[table]}`).all(cutoff);
157
+ }
158
+ return {
159
+ schema: schema_1.LEARNING_SCHEMA_NAME,
160
+ version: schema_1.LEARNING_SCHEMA_VERSION,
161
+ exportedAt: new Date(this.now()).toISOString(),
162
+ retentionDays: this.retentionDays,
163
+ tables,
164
+ };
165
+ }
166
+ /** Row counts inside the retention window. */
167
+ counts() {
168
+ this.maybePrune();
169
+ const cutoff = this.cutoff();
170
+ const count = (table) => {
171
+ const row = (table === 'candidates'
172
+ ? this.db.prepare('SELECT COUNT(*) AS n FROM candidates WHERE retrieval_id IN (SELECT id FROM retrievals WHERE at >= ?)').get(cutoff)
173
+ : this.db.prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE at >= ?`).get(cutoff));
174
+ return row.n;
175
+ };
176
+ return Object.fromEntries(schema_1.LEARNING_TABLES.map(table => [table, count(table)]));
177
+ }
178
+ cutoff() {
179
+ return this.now() - this.retentionDays * DAY_MS;
180
+ }
181
+ /**
182
+ * Removes every row and replaces the key secret, then shrinks the file. A running server keeps its open handle: it writes
183
+ * into the same file, under the new secret after a restart, and the deleted rows are gone from the file either way.
184
+ */
185
+ deleteAll() {
186
+ const secret = (0, node_crypto_1.randomBytes)(32);
187
+ this.db.transaction(() => {
188
+ for (const table of ['candidates', 'retrievals', 'memory_ops', 'feedback', 'outcomes']) {
189
+ this.db.prepare(`DELETE FROM ${table}`).run();
190
+ }
191
+ this.db.prepare(`UPDATE meta SET value = ? WHERE key = 'secret'`).run(secret.toString('hex'));
192
+ })();
193
+ this.secret = secret;
194
+ try {
195
+ this.db.pragma('wal_checkpoint(TRUNCATE)');
196
+ this.db.exec('VACUUM');
197
+ }
198
+ catch {
199
+ // Another connection holds the file. The rows are gone and secure_delete has overwritten them.
200
+ }
201
+ }
202
+ /**
203
+ * Removes every link to a permanently erased memory. The erase operation itself is deliberately
204
+ * not keyed: retaining the old HMAC beside its secret would let a later holder confirm a guess
205
+ * about data the owner erased.
206
+ */
207
+ eraseMemory(memoryKey) {
208
+ this.write(() => {
209
+ this.db.prepare('DELETE FROM candidates WHERE memory_key = ?').run(memoryKey);
210
+ this.db.prepare('DELETE FROM feedback WHERE memory_key = ?').run(memoryKey);
211
+ this.db.prepare('DELETE FROM memory_ops WHERE memory_key = ? OR replaced_key = ?').run(memoryKey, memoryKey);
212
+ });
213
+ }
214
+ close() {
215
+ if (this.db.open)
216
+ this.db.close();
217
+ }
218
+ write(fn) {
219
+ this.maybePrune();
220
+ return this.db.transaction(fn)();
221
+ }
222
+ maybePrune() {
223
+ if (this.now() - this.lastPrune >= PRUNE_EVERY_MS)
224
+ this.prune();
225
+ }
226
+ }
227
+ exports.LearningLog = LearningLog;
228
+ /** Reads the meta table, makes it on first use, and returns the key secret. Refuses a file that is not this log's version. */
229
+ function ensureMeta(db, now) {
230
+ const rows = db.prepare('SELECT key, value FROM meta').all();
231
+ const meta = new Map(rows.map(row => [row.key, row.value]));
232
+ if (meta.size === 0) {
233
+ const secret = (0, node_crypto_1.randomBytes)(32).toString('hex');
234
+ db.transaction(() => {
235
+ const put = db.prepare('INSERT INTO meta (key, value) VALUES (?, ?)');
236
+ put.run('schema', schema_1.LEARNING_SCHEMA_NAME);
237
+ put.run('version', String(schema_1.LEARNING_SCHEMA_VERSION));
238
+ put.run('created_at', new Date(now).toISOString());
239
+ put.run('secret', secret);
240
+ })();
241
+ return Buffer.from(secret, 'hex');
242
+ }
243
+ if (meta.get('schema') !== schema_1.LEARNING_SCHEMA_NAME || meta.get('version') !== String(schema_1.LEARNING_SCHEMA_VERSION)) {
244
+ throw new Error(`the learning log file is not a version-${schema_1.LEARNING_SCHEMA_VERSION} Renor learning log; refusing to use it`);
245
+ }
246
+ const secret = Buffer.from(meta.get('secret') ?? '', 'hex');
247
+ if (secret.length !== 32)
248
+ throw new Error('the learning log has no usable key secret; refusing to use it');
249
+ return secret;
250
+ }
251
+ const open = new Map();
252
+ /** The one log a process writes per file: the MCP sessions of one memory folder share it. */
253
+ function learningLogAt(settings) {
254
+ const cached = open.get(settings.dbPath);
255
+ if (cached)
256
+ return cached;
257
+ const log = LearningLog.open(settings.dbPath, { retentionDays: settings.retentionDays, rawQueries: settings.rawQueries, create: true });
258
+ if (log === null)
259
+ throw new Error('the learning log could not be created');
260
+ open.set(settings.dbPath, log);
261
+ return log;
262
+ }
263
+ function closeLearningLogs() {
264
+ for (const log of open.values())
265
+ log.close();
266
+ open.clear();
267
+ }