renor-engine 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +13 -8
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/api/server.js +10 -8
  6. package/dist/buildIdentity.js +18 -1
  7. package/dist/callPhases.js +82 -0
  8. package/dist/cli.js +36 -1
  9. package/dist/context/rowStore.js +1 -1
  10. package/dist/conversation/parser.js +40 -36
  11. package/dist/db/client.js +6 -4
  12. package/dist/db/handoffTable.js +31 -0
  13. package/dist/db/indexSalvage.js +24 -4
  14. package/dist/db/migrations.js +30 -2
  15. package/dist/db/sqlite.js +396 -0
  16. package/dist/digests/digestCache.js +6 -2
  17. package/dist/digests/digests.js +3 -3
  18. package/dist/digests/worker.js +3 -1
  19. package/dist/embeddings/caching.js +5 -5
  20. package/dist/embeddings/local.js +5 -3
  21. package/dist/embeddings/openai.js +4 -3
  22. package/dist/embeddings/simdScan.js +3 -1
  23. package/dist/embeddings/vectorIndex.js +0 -1
  24. package/dist/embeddings/vertex.js +123 -13
  25. package/dist/eval/cacheReport.js +7 -0
  26. package/dist/eval/context700-candidate.js +5 -5
  27. package/dist/eval/evalCache.js +46 -3
  28. package/dist/eval/indexSnapshots.js +549 -0
  29. package/dist/eval/pageDump.js +67 -0
  30. package/dist/eval/runManifest.js +6 -0
  31. package/dist/eval/textTokens.js +17 -0
  32. package/dist/evidence/attach.js +0 -3
  33. package/dist/evidence/evidenceCache.js +3 -3
  34. package/dist/evidence/ollamaChat.js +1 -1
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/jevDecisionModel.js +307 -0
  39. package/dist/extraction/preferenceExtractor.js +1 -4
  40. package/dist/extraction/secretGuard.js +7 -3
  41. package/dist/extraction/typedFacts.js +1 -1
  42. package/dist/importing/apply.js +42 -14
  43. package/dist/importing/cli.js +9 -8
  44. package/dist/importing/existing.js +2 -0
  45. package/dist/importing/facts.js +0 -2
  46. package/dist/importing/plan.js +0 -5
  47. package/dist/importing/source.js +1 -1
  48. package/dist/indexer/parser.js +4 -1
  49. package/dist/indexer/pipeline.js +99 -279
  50. package/dist/indexer/walker.js +11 -25
  51. package/dist/learning/cli.js +113 -0
  52. package/dist/learning/schema.js +144 -0
  53. package/dist/learning/settings.js +45 -0
  54. package/dist/learning/store.js +267 -0
  55. package/dist/learning/tap.js +401 -0
  56. package/dist/mcp/server.js +36 -6
  57. package/dist/memory/dreamer/adjudicate.js +215 -0
  58. package/dist/memory/dreamer/apply.js +120 -0
  59. package/dist/memory/dreamer/budget.js +64 -0
  60. package/dist/memory/dreamer/candidates.js +149 -0
  61. package/dist/memory/dreamer/collection.js +104 -0
  62. package/dist/memory/dreamer/dreamer.js +352 -0
  63. package/dist/memory/dreamer/review.js +111 -0
  64. package/dist/memory/factConflicts.js +2 -0
  65. package/dist/memory/handoff.js +363 -0
  66. package/dist/memory/lifecycle.js +338 -80
  67. package/dist/memory/sourceName.js +77 -1
  68. package/dist/memory/statusFacts.js +61 -0
  69. package/dist/memory/writer.js +3 -2
  70. package/dist/memorymap/memoryMap.js +39 -13
  71. package/dist/memorymap/replacements.js +8 -3
  72. package/dist/mvp/agentMemory.js +109 -50
  73. package/dist/mvp/answerCheck.js +211 -0
  74. package/dist/mvp/cli.js +14 -1
  75. package/dist/mvp/compileCache.js +3 -10
  76. package/dist/mvp/compileSeed.js +9 -3
  77. package/dist/mvp/config.js +4 -0
  78. package/dist/mvp/contextDistinct.js +53 -0
  79. package/dist/mvp/contextProfile.js +87 -0
  80. package/dist/mvp/directorySurface.js +68 -0
  81. package/dist/mvp/engine.js +39 -26
  82. package/dist/mvp/exportAll.js +36 -2
  83. package/dist/mvp/factMarkers.js +18 -2
  84. package/dist/mvp/history.js +5 -4
  85. package/dist/mvp/log.js +118 -0
  86. package/dist/mvp/server.js +759 -136
  87. package/dist/mvp/serverCard.js +12 -3
  88. package/dist/mvp/telemetry.js +35 -2
  89. package/dist/net/poolFetch.js +174 -0
  90. package/dist/recall/actionRecall.js +282 -0
  91. package/dist/recall/autoRecall.js +447 -0
  92. package/dist/recall/searchRecall.js +189 -0
  93. package/dist/retrieval/entityLookup.js +53 -4
  94. package/dist/retrieval/orchestrator.js +148 -10
  95. package/dist/retrieval/plainWording.js +236 -0
  96. package/dist/retrieval/rankingTrace.js +184 -0
  97. package/dist/retrieval/readerLayout.js +143 -224
  98. package/dist/router/routerCache.js +2 -2
  99. package/dist/sharing/cli.js +181 -0
  100. package/dist/sharing/examples.js +136 -0
  101. package/dist/sharing/fields.js +225 -0
  102. package/dist/sharing/loop.js +80 -0
  103. package/dist/sharing/project.js +327 -0
  104. package/dist/sharing/send.js +466 -0
  105. package/dist/sharing/state.js +222 -0
  106. package/dist/sharing/wording.js +48 -0
  107. package/dist/watcher/watcher.js +5 -3
  108. package/package.json +5 -7
  109. package/dist/BUILD.json +0 -6
  110. package/dist/importing/index.js +0 -29
@@ -0,0 +1,113 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.run = run;
7
+ const node_fs_1 = __importDefault(require("node:fs"));
8
+ const settings_1 = require("./settings");
9
+ const store_1 = require("./store");
10
+ /**
11
+ * Manage one memory folder's learning log: `status`, `export`, `prune` and `delete-all`. They work whether or not recording is
12
+ * on, so a log written before it was switched off can still be read or deleted, and they never create a log that does not exist.
13
+ *
14
+ * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts status
15
+ * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts export [--out <file>] (the file is written owner-only)
16
+ * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts prune
17
+ * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts delete-all --yes
18
+ *
19
+ * Stop a running server before deleting, as the data-deletion runbook does; a running server would otherwise keep its open
20
+ * handle to the file. The counts and the export never contain a key secret or a memory's text.
21
+ */
22
+ const USAGE = 'usage: learning-cli <status | export [--out <file>] | prune | delete-all --yes>';
23
+ function print(value) {
24
+ process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
25
+ }
26
+ function run(argv, env = process.env) {
27
+ const memoryDir = (env.MVP_MEMORY_DIR ?? '').trim();
28
+ if (memoryDir === '')
29
+ throw new Error('MVP_MEMORY_DIR is required: the memory folder whose learning log this manages');
30
+ const [command, ...rest] = argv;
31
+ const file = (0, settings_1.learningDbPath)(memoryDir);
32
+ const open = () => {
33
+ const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
34
+ if (log === null)
35
+ throw new Error(`no learning log at ${file}: nothing has been recorded there`);
36
+ return log;
37
+ };
38
+ switch (command) {
39
+ case 'status': {
40
+ const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
41
+ if (log === null) {
42
+ print({ present: false, file });
43
+ return;
44
+ }
45
+ try {
46
+ print({ present: true, file, retentionDays: (0, settings_1.learningRetentionDays)(env), counts: log.counts() });
47
+ }
48
+ finally {
49
+ log.close();
50
+ }
51
+ return;
52
+ }
53
+ case 'export': {
54
+ const out = rest[0] === '--out' ? rest[1] : undefined;
55
+ if (rest[0] === '--out' && (out === undefined || out === ''))
56
+ throw new Error(USAGE);
57
+ const log = open();
58
+ let text;
59
+ try {
60
+ text = `${JSON.stringify(log.exportLog(), null, 2)}\n`;
61
+ }
62
+ finally {
63
+ log.close();
64
+ }
65
+ if (out === undefined) {
66
+ process.stdout.write(text);
67
+ }
68
+ else {
69
+ node_fs_1.default.writeFileSync(out, text, { mode: 0o600 });
70
+ process.stderr.write(`wrote ${out}\n`);
71
+ }
72
+ return;
73
+ }
74
+ case 'prune': {
75
+ const log = open();
76
+ try {
77
+ print(log.prune());
78
+ }
79
+ finally {
80
+ log.close();
81
+ }
82
+ return;
83
+ }
84
+ case 'delete-all': {
85
+ if (rest[0] !== '--yes')
86
+ throw new Error('delete-all removes every recorded row and needs --yes');
87
+ const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
88
+ if (log === null) {
89
+ print({ deleted: false, file, reason: 'no learning log' });
90
+ return;
91
+ }
92
+ try {
93
+ log.deleteAll();
94
+ print({ deleted: true, file, counts: log.counts() });
95
+ }
96
+ finally {
97
+ log.close();
98
+ }
99
+ return;
100
+ }
101
+ default:
102
+ throw new Error(USAGE);
103
+ }
104
+ }
105
+ if (require.main === module) {
106
+ try {
107
+ run(process.argv.slice(2));
108
+ }
109
+ catch (err) {
110
+ process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
111
+ process.exitCode = 1;
112
+ }
113
+ }
@@ -0,0 +1,144 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.LEARNING_DDL = exports.LEARNING_TABLES = exports.TERM_COLUMNS = exports.OUTCOME_BASES = exports.OUTCOME_VALUES = exports.CITE_TARGETS = exports.OPERATIONS = exports.OPERATION_KINDS = exports.CANDIDATE_KINDS = exports.ACTORS = exports.SAVE_TOOLS = exports.WRITE_TOOLS = exports.RETRIEVAL_TOOLS = exports.RANKER_BASELINE = exports.LEARNING_SCHEMA_VERSION = exports.LEARNING_SCHEMA_NAME = void 0;
4
+ const rankingTrace_1 = require("../retrieval/rankingTrace");
5
+ /**
6
+ * The learning log's on-disk contract, version 2 (docs/learning/FEEDBACK-LOG.md is the prose version).
7
+ *
8
+ * The log holds identifiers, keyed hashes and numbers about what the agent retrieved, what it wrote, what it cited and what
9
+ * outcomes were reported. It never holds memory text, a subject or predicate in clear, a path in clear, a credential or a
10
+ * query in clear. The one exception is `query_text`, which exists only when RENOR_LEARNING_RAW_QUERY is on, and then only when
11
+ * the secret scanner does not flag the query (`query_redacted` says when it did).
12
+ *
13
+ * Every `*_key` column is a keyed hash (HMAC-SHA256, 128 bits) under a secret that is random per log and never exported:
14
+ * the same memory, query or session gets the same key within one log, and different keys in any other log.
15
+ *
16
+ * Version 2 adds, for every candidate the ranker scored or presented, its base score and one numeric column per additive term
17
+ * the ranker applied (`term_*`, from RANKING_TERMS), so that base + the terms is the score it ranked by and the weights can be
18
+ * refitted from the log. It also records the candidate's rank in the scored pool, its position in the presented window and
19
+ * whether the budget kept it. Version 1 was never released; a version 1 file is refused, not migrated.
20
+ */
21
+ exports.LEARNING_SCHEMA_NAME = 'renor-learning-log';
22
+ exports.LEARNING_SCHEMA_VERSION = 2;
23
+ /** The ranking procedure that ordered a reply. The shipped product ranking is `baseline`; a learned ranker writes its own name. */
24
+ exports.RANKER_BASELINE = 'baseline';
25
+ /** Tools whose successful replies are recorded as retrievals (one row each, with its candidates). */
26
+ exports.RETRIEVAL_TOOLS = ['get_context', 'search_memory', 'retrieve_memory'];
27
+ /** Tools whose successful calls are recorded as memory operations or, for the save tools, as explicit citations too. */
28
+ exports.WRITE_TOOLS = [
29
+ 'remember', 'save_memory', 'remember_note', 'retire_memory', 'restore_memory', 'delete_retired_memory', 'read_memory',
30
+ ];
31
+ exports.SAVE_TOOLS = ['remember', 'save_memory', 'remember_note'];
32
+ /** Who made the call: the agent over MCP, or the owner's dashboard (the platform acting for the owner). */
33
+ exports.ACTORS = ['agent', 'dashboard'];
34
+ /** What a candidate or an operation refers to. `unkeyed`: the reply named nothing a memory key can be made from. */
35
+ exports.CANDIDATE_KINDS = ['fact', 'note', 'unkeyed'];
36
+ exports.OPERATION_KINDS = ['fact', 'note', 'event', 'unkeyed'];
37
+ exports.OPERATIONS = ['create', 'supersede', 'delete', 'restore', 'erase', 'retrieve'];
38
+ exports.CITE_TARGETS = ['note', 'event', 'subject', 'unkeyed'];
39
+ exports.OUTCOME_VALUES = ['success', 'failure', 'partial', 'unknown'];
40
+ /**
41
+ * Where an outcome came from. `verifier` (a check that ran) and `user` (the owner said so) are observed. `proxy` is anything
42
+ * inferred from behaviour, such as "no correction followed"; it is stored with observed = 0 and is never a success by itself.
43
+ */
44
+ exports.OUTCOME_BASES = ['verifier', 'user', 'proxy'];
45
+ /** The column of each ranker term in `candidates`: `term_<name>`, in RANKING_TERMS order. */
46
+ exports.TERM_COLUMNS = rankingTrace_1.RANKING_TERMS.map(term => `term_${term}`);
47
+ /** The tables, in the order an export lists them. `meta` holds the schema name, its version, the creation instant and the key secret. */
48
+ exports.LEARNING_TABLES = ['retrievals', 'candidates', 'memory_ops', 'feedback', 'outcomes'];
49
+ const list = (values) => values.map(value => `'${value}'`).join(', ');
50
+ exports.LEARNING_DDL = `
51
+ CREATE TABLE IF NOT EXISTS meta (
52
+ key TEXT PRIMARY KEY,
53
+ value TEXT NOT NULL
54
+ ) WITHOUT ROWID;
55
+
56
+ CREATE TABLE IF NOT EXISTS retrievals (
57
+ id INTEGER PRIMARY KEY,
58
+ at INTEGER NOT NULL, -- unix ms
59
+ tool TEXT NOT NULL CHECK (tool IN (${list(exports.RETRIEVAL_TOOLS)})),
60
+ actor TEXT NOT NULL CHECK (actor IN (${list(exports.ACTORS)})),
61
+ session_key TEXT, -- keyed hash of the MCP session; NULL when the server keeps none
62
+ agent_key TEXT, -- keyed hash of the agent name the connection presented ('unnamed' when none)
63
+ query_key TEXT, -- keyed hash of the normalized query; NULL when the call had none
64
+ query_text TEXT, -- the raw query: RENOR_LEARNING_RAW_QUERY only, never when query_redacted = 1
65
+ query_redacted INTEGER NOT NULL DEFAULT 0 CHECK (query_redacted IN (0, 1)),
66
+ scope_key TEXT, -- keyed hash of the normalized scope ('global' when none was named)
67
+ retrieval TEXT, -- the reply's retrieval mode, when it names one
68
+ ranker TEXT NOT NULL, -- which procedure ordered the candidates (RANKER_BASELINE for the shipped one)
69
+ model TEXT, -- embedding model identity (<provider>:<model>:<dims>); NULL when keyword only
70
+ build TEXT, -- commit of the running build, NULL when unstamped
71
+ served INTEGER NOT NULL, -- candidates the reply served (rows in candidates with a rank)
72
+ ms REAL NOT NULL, -- wall time of the handler
73
+ reply_tokens INTEGER, -- o200k tokens of the whole reply, as the max_tokens budget counts them
74
+ relevant INTEGER CHECK (relevant IS NULL OR relevant IN (0, 1)), -- the engine's own verdict, when it judged
75
+ truncated INTEGER CHECK (truncated IS NULL OR truncated IN (0, 1)),
76
+ lean_items INTEGER NOT NULL DEFAULT 0 CHECK (lean_items IN (0, 1)) -- RENOR_CONTEXT_LEAN_ITEMS was on: no note paths in the reply
77
+ );
78
+
79
+ CREATE TABLE IF NOT EXISTS candidates (
80
+ id INTEGER PRIMARY KEY,
81
+ retrieval_id INTEGER NOT NULL REFERENCES retrievals(id) ON DELETE CASCADE,
82
+ rank INTEGER, -- 1-based position in the reply (facts first, then memories, as served); NULL when not served
83
+ kind TEXT NOT NULL CHECK (kind IN (${list(exports.CANDIDATE_KINDS)})),
84
+ memory_key TEXT, -- fact: scope + subject + predicate; note: the note path
85
+ subject_key TEXT, -- fact only: the subject within its scope (joins cites that name it, in that scope)
86
+ section_key TEXT, -- note only: the heading path inside the note, when there is one
87
+ score REAL, -- the score the candidate was ranked by: the reply's, or base + terms when not served
88
+ base REAL, -- the score before any term: a cosine, 0.3 for a graph hit, the fact lane's, 0 for a statement; NULL when untraced
89
+ ${exports.TERM_COLUMNS.map(column => `${column} REAL NOT NULL DEFAULT 0,`).join('\n ')}
90
+ pool_rank INTEGER, -- 1-based rank in the scored pool (score order, before the window cut); NULL when not scored
91
+ window_pos INTEGER, -- 1-based position in the order the reply presents its hits, before any budget cut; NULL when not presented
92
+ budget_kept INTEGER CHECK (budget_kept IS NULL OR budget_kept IN (0, 1)), -- 1 kept by max_tokens, 0 cut; NULL when the read had no budget
93
+ reason TEXT, -- the lane that matched (fact, fts, semantic, semantic+content+recency, ...)
94
+ matched INTEGER CHECK (matched IS NULL OR matched IN (0, 1)), -- 0: the reply marked it as not matched
95
+ current INTEGER CHECK (current IS NULL OR current IN (0, 1)) -- retrieve_memory: 0 for a superseded value
96
+ );
97
+
98
+ CREATE TABLE IF NOT EXISTS memory_ops (
99
+ id INTEGER PRIMARY KEY,
100
+ at INTEGER NOT NULL,
101
+ actor TEXT NOT NULL CHECK (actor IN (${list(exports.ACTORS)})),
102
+ session_key TEXT,
103
+ agent_key TEXT,
104
+ tool TEXT NOT NULL CHECK (tool IN (${list(exports.WRITE_TOOLS)})),
105
+ op TEXT NOT NULL CHECK (op IN (${list(exports.OPERATIONS)})),
106
+ kind TEXT NOT NULL CHECK (kind IN (${list(exports.OPERATION_KINDS)})),
107
+ memory_key TEXT,
108
+ subject_key TEXT,
109
+ replaced_key TEXT, -- supersede only: the memory identity whose value the new save replaces
110
+ ms REAL NOT NULL
111
+ );
112
+
113
+ CREATE TABLE IF NOT EXISTS feedback (
114
+ id INTEGER PRIMARY KEY,
115
+ at INTEGER NOT NULL,
116
+ actor TEXT NOT NULL CHECK (actor IN (${list(exports.ACTORS)})),
117
+ session_key TEXT,
118
+ agent_key TEXT,
119
+ tool TEXT NOT NULL CHECK (tool IN (${list(exports.SAVE_TOOLS)})),
120
+ kind TEXT NOT NULL CHECK (kind IN ('cite')), -- the explicit cites argument of a save
121
+ target TEXT NOT NULL CHECK (target IN (${list(exports.CITE_TARGETS)})),
122
+ memory_key TEXT,
123
+ subject_key TEXT,
124
+ observed INTEGER NOT NULL DEFAULT 1 CHECK (observed IN (0, 1))
125
+ );
126
+
127
+ CREATE TABLE IF NOT EXISTS outcomes (
128
+ id INTEGER PRIMARY KEY,
129
+ at INTEGER NOT NULL,
130
+ session_key TEXT,
131
+ task_key TEXT, -- keyed hash of an opaque task reference the caller supplies; NULL when none
132
+ outcome TEXT NOT NULL CHECK (outcome IN (${list(exports.OUTCOME_VALUES)})),
133
+ basis TEXT NOT NULL CHECK (basis IN (${list(exports.OUTCOME_BASES)})),
134
+ observed INTEGER NOT NULL CHECK (observed IN (0, 1)) -- 1 unless basis = proxy
135
+ );
136
+
137
+ CREATE INDEX IF NOT EXISTS retrievals_at ON retrievals (at);
138
+ CREATE INDEX IF NOT EXISTS candidates_retrieval ON candidates (retrieval_id, rank);
139
+ CREATE INDEX IF NOT EXISTS candidates_memory ON candidates (memory_key);
140
+ CREATE INDEX IF NOT EXISTS memory_ops_at ON memory_ops (at);
141
+ CREATE INDEX IF NOT EXISTS memory_ops_memory ON memory_ops (memory_key);
142
+ CREATE INDEX IF NOT EXISTS feedback_at ON feedback (at);
143
+ CREATE INDEX IF NOT EXISTS outcomes_at ON outcomes (at);
144
+ `;
@@ -0,0 +1,45 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.MAX_LEARNING_RETENTION_DAYS = exports.DEFAULT_LEARNING_RETENTION_DAYS = void 0;
7
+ exports.learningDbPath = learningDbPath;
8
+ exports.learningRetentionDays = learningRetentionDays;
9
+ exports.learningSettings = learningSettings;
10
+ const node_path_1 = __importDefault(require("node:path"));
11
+ /**
12
+ * Learning-log settings, read from the environment. Off by default: with nothing set, no file is made and no call is recorded.
13
+ * A bad value is an error at start-up, as RENOR_ON_DEMAND is, so a typo can never leave the log on or off by accident.
14
+ */
15
+ exports.DEFAULT_LEARNING_RETENTION_DAYS = 30;
16
+ exports.MAX_LEARNING_RETENTION_DAYS = 365;
17
+ /** The learning log's file for one memory folder. Each tenant has its own memory folder, so each tenant has its own log. */
18
+ function learningDbPath(memoryDir) {
19
+ return node_path_1.default.join(node_path_1.default.resolve(memoryDir), '.memory-index', 'learning.sqlite');
20
+ }
21
+ function onOff(name, raw) {
22
+ const value = (raw ?? '').trim().toLowerCase();
23
+ if (value === '' || value === '0' || value === 'false')
24
+ return false;
25
+ if (value === '1' || value === 'true')
26
+ return true;
27
+ throw new Error(`${name} must be 1/true or 0/false`);
28
+ }
29
+ /** RENOR_LEARNING_RETENTION_DAYS, validated. The management commands read it too, whether or not the log is on. */
30
+ function learningRetentionDays(env = process.env) {
31
+ const raw = (env.RENOR_LEARNING_RETENTION_DAYS ?? '').trim();
32
+ const days = raw === '' ? exports.DEFAULT_LEARNING_RETENTION_DAYS : Number(raw);
33
+ if (!Number.isInteger(days) || days < 1 || days > exports.MAX_LEARNING_RETENTION_DAYS) {
34
+ throw new Error(`RENOR_LEARNING_RETENTION_DAYS must be an integer between 1 and ${exports.MAX_LEARNING_RETENTION_DAYS}`);
35
+ }
36
+ return days;
37
+ }
38
+ function learningSettings(memoryDir, env = process.env) {
39
+ const enabled = onOff('RENOR_LEARNING_LOG', env.RENOR_LEARNING_LOG);
40
+ const rawQueries = onOff('RENOR_LEARNING_RAW_QUERY', env.RENOR_LEARNING_RAW_QUERY);
41
+ if (rawQueries && !enabled) {
42
+ throw new Error('RENOR_LEARNING_RAW_QUERY needs RENOR_LEARNING_LOG=1: a query is only ever kept by a log that is on');
43
+ }
44
+ return { enabled, retentionDays: learningRetentionDays(env), rawQueries, dbPath: learningDbPath(memoryDir) };
45
+ }
@@ -0,0 +1,267 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.LearningLog = void 0;
7
+ exports.learningLogAt = learningLogAt;
8
+ exports.closeLearningLogs = closeLearningLogs;
9
+ const sqlite_1 = __importDefault(require("../db/sqlite"));
10
+ const node_crypto_1 = require("node:crypto");
11
+ const node_fs_1 = __importDefault(require("node:fs"));
12
+ const node_path_1 = __importDefault(require("node:path"));
13
+ const secretScanner_1 = require("../extraction/secretScanner");
14
+ const schema_1 = require("./schema");
15
+ /**
16
+ * The learning log: one SQLite file per memory folder, written by the MCP tools when RENOR_LEARNING_LOG is on (tap.ts) and
17
+ * read by the learner through `exportLog()`. It is separate from product analytics (telemetry.ts) and from the index, so
18
+ * deleting the index never deletes it and deleting the log never touches memory.
19
+ *
20
+ * The secret that makes the keys is written once, in the `meta` table, when the log is created; `exportLog()` never includes
21
+ * it, and `deleteAll()` replaces it, so keys made before a delete cannot be matched to keys made after one.
22
+ */
23
+ const DAY_MS = 86_400_000;
24
+ const PRUNE_EVERY_MS = 3_600_000;
25
+ /** The order an export lists each table in: ids where a table has one, otherwise its key. */
26
+ const ORDER = {
27
+ retrievals: 'id',
28
+ candidates: 'retrieval_id, id',
29
+ memory_ops: 'id',
30
+ feedback: 'id',
31
+ outcomes: 'id',
32
+ };
33
+ const flag = (value) => (value === null ? null : value ? 1 : 0);
34
+ const OUTCOME_OBSERVED = { verifier: 1, user: 1, proxy: 0 };
35
+ class LearningLog {
36
+ file;
37
+ db;
38
+ retentionDays;
39
+ rawQueries;
40
+ now;
41
+ secret;
42
+ lastPrune;
43
+ constructor(file, db, secret, retentionDays, rawQueries, now) {
44
+ this.file = file;
45
+ this.db = db;
46
+ this.retentionDays = retentionDays;
47
+ this.rawQueries = rawQueries;
48
+ this.now = now;
49
+ this.secret = secret;
50
+ // Zero makes the first read or write enforce retention immediately.
51
+ this.lastPrune = 0;
52
+ }
53
+ /** Opens the log at `file`, creating it (owner-only) when `create` is set. Returns null when there is no log and none may be made. */
54
+ static open(file, options) {
55
+ const now = options.now ?? Date.now;
56
+ if (options.create !== true && !node_fs_1.default.existsSync(file))
57
+ return null;
58
+ // Owner-only, like the index (db/client.ts). Tighten an existing file too: it may have been
59
+ // restored with a permissive umask and it contains linkable pseudonymous activity.
60
+ node_fs_1.default.mkdirSync(node_path_1.default.dirname(file), { recursive: true, mode: 0o700 });
61
+ node_fs_1.default.closeSync(node_fs_1.default.openSync(file, node_fs_1.default.constants.O_CREAT | node_fs_1.default.constants.O_WRONLY, 0o600));
62
+ node_fs_1.default.chmodSync(file, 0o600);
63
+ const db = new sqlite_1.default(file);
64
+ try {
65
+ db.pragma('journal_mode = WAL');
66
+ db.pragma('foreign_keys = ON');
67
+ // Deleted rows are overwritten, not left in free pages: a delete means the content is gone from the file.
68
+ db.pragma('secure_delete = ON');
69
+ db.pragma('busy_timeout = 5000');
70
+ db.exec(schema_1.LEARNING_DDL);
71
+ const secret = ensureMeta(db, now());
72
+ return new LearningLog(file, db, secret, options.retentionDays, options.rawQueries, now);
73
+ }
74
+ catch (err) {
75
+ db.close();
76
+ throw err;
77
+ }
78
+ }
79
+ /**
80
+ * A keyed hash of `parts` under this log's secret, 128 bits in hex. `kind` separates the namespaces (memory, query, session...);
81
+ * the parts are JSON-encoded, so ["a b"] and ["a", "b"] cannot collide.
82
+ */
83
+ key(kind, ...parts) {
84
+ return (0, node_crypto_1.createHmac)('sha256', this.secret).update(JSON.stringify([kind, ...parts])).digest('hex').slice(0, 32);
85
+ }
86
+ recordRetrieval(record) {
87
+ const at = record.at ?? this.now();
88
+ // The raw query is written only by a log opened with rawQueries, and never when the secret scanner flags it: the check
89
+ // lives here, at the write, so no caller can keep a query by passing it in.
90
+ // query_redacted means one thing: the scanner flagged a query that the opt-in would have kept.
91
+ const raw = record.queryText;
92
+ const flagged = raw !== null && this.rawQueries && (0, secretScanner_1.scanSecrets)(raw) !== null;
93
+ const keepRaw = raw !== null && this.rawQueries && !flagged;
94
+ const queryRedacted = record.queryRedacted || flagged;
95
+ return this.write(() => {
96
+ const info = this.db.prepare(`INSERT INTO retrievals (at, tool, actor, session_key, agent_key, query_key, query_text, query_redacted, scope_key, retrieval,
97
+ ranker, model, build, served, ms, reply_tokens, relevant, truncated, lean_items)
98
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(at, record.tool, record.actor, record.sessionKey, record.agentKey, record.queryKey, keepRaw ? raw : null, queryRedacted ? 1 : 0, record.scopeKey, record.retrieval, record.ranker, record.model, record.build, record.candidates.filter(candidate => candidate.rank !== null).length, record.ms, record.replyTokens, flag(record.relevant), flag(record.truncated), record.leanItems ? 1 : 0);
99
+ const id = Number(info.lastInsertRowid);
100
+ const columns = [
101
+ 'retrieval_id', 'rank', 'kind', 'memory_key', 'subject_key', 'section_key', 'score', 'base', ...schema_1.TERM_COLUMNS,
102
+ 'pool_rank', 'window_pos', 'budget_kept', 'reason', 'matched', 'current',
103
+ ];
104
+ const insert = this.db.prepare(`INSERT INTO candidates (${columns.join(', ')}) VALUES (${columns.map(() => '?').join(', ')})`);
105
+ for (const candidate of record.candidates) {
106
+ const terms = candidate.terms ?? schema_1.TERM_COLUMNS.map(() => 0);
107
+ insert.run(id, candidate.rank, candidate.kind, candidate.memoryKey, candidate.subjectKey, candidate.sectionKey, candidate.score, candidate.base, ...terms, candidate.poolRank, candidate.windowPos, flag(candidate.budgetKept), candidate.reason, flag(candidate.matched), flag(candidate.current));
108
+ }
109
+ return id;
110
+ });
111
+ }
112
+ recordMemoryOp(record) {
113
+ this.write(() => {
114
+ this.db.prepare(`INSERT INTO memory_ops (at, actor, session_key, agent_key, tool, op, kind, memory_key, subject_key, replaced_key, ms)
115
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(record.at ?? this.now(), record.actor, record.sessionKey, record.agentKey, record.tool, record.op, record.kind, record.memoryKey, record.subjectKey, record.replacedKey, record.ms);
116
+ });
117
+ }
118
+ /** An explicit cite: the caller named this memory as what the save builds on. Always observed: the agent said so. */
119
+ recordFeedback(record) {
120
+ this.write(() => {
121
+ this.db.prepare(`INSERT INTO feedback (at, actor, session_key, agent_key, tool, kind, target, memory_key, subject_key, observed)
122
+ VALUES (?, ?, ?, ?, ?, 'cite', ?, ?, ?, 1)`).run(record.at ?? this.now(), record.actor, record.sessionKey, record.agentKey, record.tool, record.target, record.memoryKey, record.subjectKey);
123
+ });
124
+ }
125
+ /** An outcome of a task. `observed` follows from the basis: a proxy is inferred and is never observed. */
126
+ recordOutcome(record) {
127
+ this.write(() => {
128
+ this.db.prepare(`INSERT INTO outcomes (at, session_key, task_key, outcome, basis, observed) VALUES (?, ?, ?, ?, ?, ?)`).run(record.at ?? this.now(), record.sessionKey, record.taskRef === null ? null : this.key('task', record.taskRef), record.outcome, record.basis, OUTCOME_OBSERVED[record.basis]);
129
+ });
130
+ }
131
+ /** Deletes what is older than the retention window. Runs when the log opens and then hourly while it is written to. */
132
+ prune() {
133
+ const now = this.now();
134
+ this.lastPrune = now;
135
+ const cutoff = now - this.retentionDays * DAY_MS;
136
+ return this.db.transaction(() => {
137
+ const candidates = this.db.prepare('DELETE FROM candidates WHERE retrieval_id IN (SELECT id FROM retrievals WHERE at < ?)').run(cutoff).changes;
138
+ const retrievals = this.db.prepare('DELETE FROM retrievals WHERE at < ?').run(cutoff).changes;
139
+ const memory_ops = this.db.prepare('DELETE FROM memory_ops WHERE at < ?').run(cutoff).changes;
140
+ const feedback = this.db.prepare('DELETE FROM feedback WHERE at < ?').run(cutoff).changes;
141
+ const outcomes = this.db.prepare('DELETE FROM outcomes WHERE at < ?').run(cutoff).changes;
142
+ return { retrievals, candidates, memory_ops, feedback, outcomes };
143
+ })();
144
+ }
145
+ /**
146
+ * Every row inside the retention window, and nothing else: the key secret stays in the file and never leaves it. Rows older
147
+ * than the window are left out even before a prune has deleted them.
148
+ */
149
+ exportLog() {
150
+ this.maybePrune();
151
+ const tables = {};
152
+ const cutoff = this.cutoff();
153
+ for (const table of schema_1.LEARNING_TABLES) {
154
+ tables[table] = table === 'candidates'
155
+ ? this.db.prepare(`SELECT * FROM candidates WHERE retrieval_id IN (SELECT id FROM retrievals WHERE at >= ?) ORDER BY ${ORDER.candidates}`).all(cutoff)
156
+ : this.db.prepare(`SELECT * FROM ${table} WHERE at >= ? ORDER BY ${ORDER[table]}`).all(cutoff);
157
+ }
158
+ return {
159
+ schema: schema_1.LEARNING_SCHEMA_NAME,
160
+ version: schema_1.LEARNING_SCHEMA_VERSION,
161
+ exportedAt: new Date(this.now()).toISOString(),
162
+ retentionDays: this.retentionDays,
163
+ tables,
164
+ };
165
+ }
166
+ /** Row counts inside the retention window. */
167
+ counts() {
168
+ this.maybePrune();
169
+ const cutoff = this.cutoff();
170
+ const count = (table) => {
171
+ const row = (table === 'candidates'
172
+ ? this.db.prepare('SELECT COUNT(*) AS n FROM candidates WHERE retrieval_id IN (SELECT id FROM retrievals WHERE at >= ?)').get(cutoff)
173
+ : this.db.prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE at >= ?`).get(cutoff));
174
+ return row.n;
175
+ };
176
+ return Object.fromEntries(schema_1.LEARNING_TABLES.map(table => [table, count(table)]));
177
+ }
178
+ cutoff() {
179
+ return this.now() - this.retentionDays * DAY_MS;
180
+ }
181
+ /**
182
+ * Removes every row and replaces the key secret, then shrinks the file. A running server keeps its open handle: it writes
183
+ * into the same file, under the new secret after a restart, and the deleted rows are gone from the file either way.
184
+ */
185
+ deleteAll() {
186
+ const secret = (0, node_crypto_1.randomBytes)(32);
187
+ this.db.transaction(() => {
188
+ for (const table of ['candidates', 'retrievals', 'memory_ops', 'feedback', 'outcomes']) {
189
+ this.db.prepare(`DELETE FROM ${table}`).run();
190
+ }
191
+ this.db.prepare(`UPDATE meta SET value = ? WHERE key = 'secret'`).run(secret.toString('hex'));
192
+ })();
193
+ this.secret = secret;
194
+ try {
195
+ this.db.pragma('wal_checkpoint(TRUNCATE)');
196
+ this.db.exec('VACUUM');
197
+ }
198
+ catch {
199
+ // Another connection holds the file. The rows are gone and secure_delete has overwritten them.
200
+ }
201
+ }
202
+ /**
203
+ * Removes every link to a permanently erased memory. The erase operation itself is deliberately
204
+ * not keyed: retaining the old HMAC beside its secret would let a later holder confirm a guess
205
+ * about data the owner erased.
206
+ */
207
+ eraseMemory(memoryKey) {
208
+ this.write(() => {
209
+ this.db.prepare('DELETE FROM candidates WHERE memory_key = ?').run(memoryKey);
210
+ this.db.prepare('DELETE FROM feedback WHERE memory_key = ?').run(memoryKey);
211
+ this.db.prepare('DELETE FROM memory_ops WHERE memory_key = ? OR replaced_key = ?').run(memoryKey, memoryKey);
212
+ });
213
+ }
214
+ close() {
215
+ if (this.db.open)
216
+ this.db.close();
217
+ }
218
+ write(fn) {
219
+ this.maybePrune();
220
+ return this.db.transaction(fn)();
221
+ }
222
+ maybePrune() {
223
+ if (this.now() - this.lastPrune >= PRUNE_EVERY_MS)
224
+ this.prune();
225
+ }
226
+ }
227
+ exports.LearningLog = LearningLog;
228
+ /** Reads the meta table, makes it on first use, and returns the key secret. Refuses a file that is not this log's version. */
229
+ function ensureMeta(db, now) {
230
+ const rows = db.prepare('SELECT key, value FROM meta').all();
231
+ const meta = new Map(rows.map(row => [row.key, row.value]));
232
+ if (meta.size === 0) {
233
+ const secret = (0, node_crypto_1.randomBytes)(32).toString('hex');
234
+ db.transaction(() => {
235
+ const put = db.prepare('INSERT INTO meta (key, value) VALUES (?, ?)');
236
+ put.run('schema', schema_1.LEARNING_SCHEMA_NAME);
237
+ put.run('version', String(schema_1.LEARNING_SCHEMA_VERSION));
238
+ put.run('created_at', new Date(now).toISOString());
239
+ put.run('secret', secret);
240
+ })();
241
+ return Buffer.from(secret, 'hex');
242
+ }
243
+ if (meta.get('schema') !== schema_1.LEARNING_SCHEMA_NAME || meta.get('version') !== String(schema_1.LEARNING_SCHEMA_VERSION)) {
244
+ throw new Error(`the learning log file is not a version-${schema_1.LEARNING_SCHEMA_VERSION} Renor learning log; refusing to use it`);
245
+ }
246
+ const secret = Buffer.from(meta.get('secret') ?? '', 'hex');
247
+ if (secret.length !== 32)
248
+ throw new Error('the learning log has no usable key secret; refusing to use it');
249
+ return secret;
250
+ }
251
+ const open = new Map();
252
+ /** The one log a process writes per file: the MCP sessions of one memory folder share it. */
253
+ function learningLogAt(settings) {
254
+ const cached = open.get(settings.dbPath);
255
+ if (cached)
256
+ return cached;
257
+ const log = LearningLog.open(settings.dbPath, { retentionDays: settings.retentionDays, rawQueries: settings.rawQueries, create: true });
258
+ if (log === null)
259
+ throw new Error('the learning log could not be created');
260
+ open.set(settings.dbPath, log);
261
+ return log;
262
+ }
263
+ function closeLearningLogs() {
264
+ for (const log of open.values())
265
+ log.close();
266
+ open.clear();
267
+ }