renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,181 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.TELEMETRY_ACTIONS = void 0;
7
- exports.runTelemetry = runTelemetry;
8
- const node_fs_1 = __importDefault(require("node:fs"));
9
- const settings_1 = require("../learning/settings");
10
- const store_1 = require("../learning/store");
11
- const fields_1 = require("./fields");
12
- const send_1 = require("./send");
13
- const state_1 = require("./state");
14
- const wording_1 = require("./wording");
15
- exports.TELEMETRY_ACTIONS = ['status', 'signals', 'data', 'off', 'delete', 'preview', 'send', 'fields'];
16
- function memoryDirOf(options, env) {
17
- const value = (options.memoryDir ?? env.MVP_MEMORY_DIR ?? '').trim();
18
- return value === '' ? null : value;
19
- }
20
- /** Returns the exit code. Throws nothing the user needs to read: problems are printed on `err`. */
21
- async function runTelemetry(action, argument, options, io) {
22
- const env = io.env;
23
- const memoryDir = memoryDirOf(options, env);
24
- const dir = (0, state_1.sharingDirFor)(memoryDir, env, options.dir);
25
- // Hosted or not as the service decides it, so send, off and delete reach the endpoint the way the service's sender does.
26
- const common = { dir, env, fetch: io.fetch, now: io.now, hosted: (0, state_1.hostedEngine)(env) };
27
- try {
28
- switch (action) {
29
- case 'status': {
30
- const record = (0, state_1.readConsent)(dir);
31
- const state = (0, state_1.readState)(dir);
32
- const override = (0, state_1.envOverride)(env);
33
- const level = (0, state_1.effectiveLevel)(dir, env);
34
- io.out(`sharing: ${level}${override !== null && level !== (record?.level ?? 'off') ? ` (lowered by the environment from ${record?.level ?? 'off'})` : ''}`);
35
- if (record === null)
36
- io.out('choice: none made (the default is off; nothing is recorded for sharing or sent)');
37
- else {
38
- io.out(`choice: ${record.level}, chosen ${record.changedAt} via ${record.source}, wording ${record.wording}, 18 or older: ${record.adult ? 'yes' : 'no'}`);
39
- }
40
- io.out(`folder: ${dir}`);
41
- io.out(`endpoint: ${(0, send_1.sharingEndpoint)(env)}`);
42
- if (state?.paused?.code === 'sharing_closed')
43
- io.out('sending: paused, Renor has closed sharing; nothing is sent until it is open again');
44
- else if (state?.paused?.code === 'consent_required')
45
- io.out('sending: paused, Renor refused the consent record; choose a level again to resume');
46
- io.out(`debug mode: ${(0, send_1.debugOn)(env) ? 'on (RENOR_SHARING_DEBUG): nothing is sent, bodies are printed' : 'off'}`);
47
- if (memoryDir === null)
48
- io.out('learning log: not found (set MVP_MEMORY_DIR or pass --memory-dir; sharing sends rows made from it)');
49
- else {
50
- const file = (0, settings_1.learningDbPath)(memoryDir);
51
- if (!node_fs_1.default.existsSync(file))
52
- io.out(`learning log: none at ${file}`);
53
- else {
54
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
55
- try {
56
- io.out(`learning log: ${file}`);
57
- if (log)
58
- io.out(` rows inside the retention window: ${JSON.stringify(log.counts())}`);
59
- }
60
- finally {
61
- log?.close();
62
- }
63
- }
64
- }
65
- if (state) {
66
- io.out(`sent before: ${state.everSent ? 'yes' : 'no'}${state.deletePending ? '; a deletion is owed to the endpoint and will be retried' : ''}`);
67
- }
68
- return 0;
69
- }
70
- case 'fields': {
71
- const wanted = argument === undefined ? ['signals', 'data'] : [argument];
72
- for (const level of wanted) {
73
- if (level !== 'signals' && level !== 'data') {
74
- io.err('usage: renor-engine telemetry fields [signals|data]');
75
- return 2;
76
- }
77
- io.out(`# fields sent at level ${level}`);
78
- for (const field of (0, fields_1.fieldsAtLevel)(level))
79
- io.out(field);
80
- }
81
- return 0;
82
- }
83
- case 'signals':
84
- case 'data': {
85
- const level = action;
86
- io.out((0, wording_1.wordingFor)(level));
87
- io.out('');
88
- if (options.yes !== true) {
89
- const answer = await io.ask(`${wording_1.ADULT_LINE} Share at this level? [y/N] `);
90
- if (answer === null) {
91
- io.err('No terminal to ask on: nothing was changed. Run it in a terminal, or add --yes if you have read the words above.');
92
- return 1;
93
- }
94
- if (!answer) {
95
- io.out('Nothing was changed.');
96
- return 0;
97
- }
98
- }
99
- const result = await (0, send_1.chooseLevel)({ ...common, level, wording: wording_1.CLI_WORDING_VERSION, adult: true, source: 'cli' });
100
- io.out(result.changed ? `Sharing is now: ${level}. Recorded ${result.record?.changedAt}.` : `Sharing was already: ${level}.`);
101
- if (memoryDir === null)
102
- io.out('Note: no learning log folder is set (MVP_MEMORY_DIR), so there is nothing to share yet.');
103
- return 0;
104
- }
105
- case 'off': {
106
- const result = await (0, send_1.chooseLevel)({ ...common, level: 'off', wording: wording_1.CLI_WORDING_VERSION, adult: true, source: 'cli' });
107
- if (!result.changed && result.record === null)
108
- io.out('Sharing was already off, and no choice had ever been made. Nothing was written.');
109
- else
110
- io.out('Sharing is now: off. Recording for sharing has stopped.');
111
- if (result.remoteDeleted === true)
112
- io.out('Renor was asked to delete what it holds, and has.');
113
- else if (result.deleteOwedUnderDoNotTrack)
114
- io.out('DO_NOT_TRACK=1 forbids any request, so Renor has not been asked yet to delete what it holds. The deletion is recorded here and sent the first time the engine runs without DO_NOT_TRACK=1.');
115
- else if (result.remoteDeleted === false)
116
- io.out(`Renor could not be reached to delete what it holds (${result.reason ?? 'unknown'}). The deletion is kept and retried when the service next runs.`);
117
- return result.remoteDeleted === false ? 1 : 0;
118
- }
119
- case 'delete': {
120
- const result = await (0, send_1.deleteAllShared)(common);
121
- if (!result.remote.ok) {
122
- io.err(result.remote.owedUnderDoNotTrack
123
- ? 'DO_NOT_TRACK=1 forbids any request, so Renor has not been asked yet to delete what it holds; nothing was removed here. The deletion is recorded and sent the first time the engine runs without DO_NOT_TRACK=1.'
124
- : `Renor could not be reached (${result.remote.reason ?? 'unknown'}); nothing was removed here, and the deletion is kept to retry.`);
125
- return 1;
126
- }
127
- io.out(result.remote.requested ? 'Renor deleted what it held for this install.' : 'Nothing had ever been sent, so Renor holds nothing.');
128
- io.out('The local sharing files (state, saved example texts) were removed. Your choice is unchanged.');
129
- if (options.local) {
130
- if (memoryDir === null) {
131
- io.err('--local needs MVP_MEMORY_DIR or --memory-dir: the learning log was not touched.');
132
- return 1;
133
- }
134
- const file = (0, settings_1.learningDbPath)(memoryDir);
135
- const log = node_fs_1.default.existsSync(file) ? store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false }) : null;
136
- if (log) {
137
- try {
138
- log.deleteAll();
139
- }
140
- finally {
141
- log.close();
142
- }
143
- io.out('The local learning log was emptied.');
144
- }
145
- else
146
- io.out('There was no local learning log.');
147
- }
148
- return 0;
149
- }
150
- case 'preview':
151
- case 'send': {
152
- if (action === 'preview' && argument !== undefined && argument !== 'signals' && argument !== 'data') {
153
- io.err('usage: renor-engine telemetry preview [signals|data]');
154
- return 2;
155
- }
156
- const result = await (0, send_1.shareNow)({
157
- ...common, memoryDir, debug: action === 'preview' ? true : undefined,
158
- previewLevel: action === 'preview' && (argument === 'signals' || argument === 'data') ? argument : undefined,
159
- });
160
- if (action === 'send')
161
- await (0, send_1.settlePendingDelete)(common);
162
- if (result.status === 'debug') {
163
- io.out(`# debug: nothing was sent and nothing was changed. These ${result.batches} request body(ies) are exactly what would be sent to ${(0, send_1.sharingEndpoint)(env)}:`);
164
- for (const payload of result.payloads ?? [])
165
- io.out(JSON.stringify(payload, null, 2));
166
- }
167
- else {
168
- io.out(`status: ${result.status}; level: ${result.level}; rows: ${result.rows}; batches: ${result.batches}${result.skipped ? `; skipped (failed their own contract): ${result.skipped}` : ''}${result.reason ? `; ${result.reason}` : ''}`);
169
- }
170
- return result.status === 'failed' ? 1 : 0;
171
- }
172
- default:
173
- io.err(`usage: renor-engine telemetry <${exports.TELEMETRY_ACTIONS.join(' | ')}>`);
174
- return 2;
175
- }
176
- }
177
- catch (err) {
178
- io.err(err instanceof Error ? err.message : String(err));
179
- return 1;
180
- }
181
- }
@@ -1,136 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.ExampleStore = void 0;
7
- exports.closeExampleStores = closeExampleStores;
8
- exports.sharingHook = sharingHook;
9
- const sqlite_1 = __importDefault(require("../db/sqlite"));
10
- const node_fs_1 = __importDefault(require("node:fs"));
11
- const node_path_1 = __importDefault(require("node:path"));
12
- const secretScanner_1 = require("../extraction/secretScanner");
13
- const fields_1 = require("./fields");
14
- const state_1 = require("./state");
15
- class ExampleStore {
16
- db;
17
- constructor(db) {
18
- this.db = db;
19
- }
20
- static open(dir, create) {
21
- const file = node_path_1.default.join(dir, state_1.EXAMPLES_FILE);
22
- if (!create && !node_fs_1.default.existsSync(file))
23
- return null;
24
- node_fs_1.default.mkdirSync(dir, { recursive: true, mode: 0o700 });
25
- node_fs_1.default.closeSync(node_fs_1.default.openSync(file, node_fs_1.default.constants.O_CREAT | node_fs_1.default.constants.O_WRONLY, 0o600));
26
- const db = new sqlite_1.default(file);
27
- try {
28
- db.pragma('journal_mode = WAL');
29
- db.pragma('secure_delete = ON');
30
- db.pragma('busy_timeout = 5000');
31
- db.exec(`
32
- CREATE TABLE IF NOT EXISTS queries (query_key TEXT PRIMARY KEY, text TEXT NOT NULL, at INTEGER NOT NULL) WITHOUT ROWID;
33
- CREATE TABLE IF NOT EXISTS memories (memory_key TEXT PRIMARY KEY, text TEXT NOT NULL, at INTEGER NOT NULL) WITHOUT ROWID;
34
- `);
35
- }
36
- catch (err) {
37
- db.close();
38
- throw err;
39
- }
40
- return new ExampleStore(db);
41
- }
42
- putQuery(queryKey, text, at) {
43
- const kept = keepable(text);
44
- if (kept === null)
45
- return;
46
- this.db.prepare('INSERT INTO queries (query_key, text, at) VALUES (?, ?, ?) ON CONFLICT (query_key) DO UPDATE SET text = excluded.text, at = excluded.at')
47
- .run(queryKey, kept, at);
48
- }
49
- putMemory(memoryKey, text, at) {
50
- const kept = keepable(text);
51
- if (kept === null)
52
- return;
53
- this.db.prepare('INSERT INTO memories (memory_key, text, at) VALUES (?, ?, ?) ON CONFLICT (memory_key) DO UPDATE SET text = excluded.text, at = excluded.at')
54
- .run(memoryKey, kept, at);
55
- }
56
- query(queryKey) {
57
- const row = this.db.prepare('SELECT text FROM queries WHERE query_key = ?').get(queryKey);
58
- return row?.text ?? null;
59
- }
60
- memory(memoryKey) {
61
- const row = this.db.prepare('SELECT text FROM memories WHERE memory_key = ?').get(memoryKey);
62
- return row?.text ?? null;
63
- }
64
- count() {
65
- const n = (table) => this.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n;
66
- return { queries: n('queries'), memories: n('memories') };
67
- }
68
- /** Deletes the texts last put at or before `cutoff` (ms): after a full send, no unsent row uses them. */
69
- forgetUpTo(cutoff) {
70
- this.db.prepare('DELETE FROM queries WHERE at <= ?').run(cutoff);
71
- this.db.prepare('DELETE FROM memories WHERE at <= ?').run(cutoff);
72
- }
73
- close() {
74
- if (this.db.open)
75
- this.db.close();
76
- }
77
- }
78
- exports.ExampleStore = ExampleStore;
79
- /** A text worth keeping: not empty, not flagged by the secret scanner; cut at the limit. */
80
- function keepable(text) {
81
- const trimmed = text.trim();
82
- if (trimmed === '')
83
- return null;
84
- const cut = trimmed.length > fields_1.MAX_EXAMPLE_TEXT ? trimmed.slice(0, fields_1.MAX_EXAMPLE_TEXT) : trimmed;
85
- // The scan covers the whole text, not just the part kept: a secret past the cut still marks the text as one to leave out.
86
- return (0, secretScanner_1.scanSecrets)(trimmed) === null ? cut : null;
87
- }
88
- const open = new Map();
89
- function closeExampleStores() {
90
- for (const store of open.values())
91
- store.close();
92
- open.clear();
93
- }
94
- /** The hook for one sharing folder. `now` is for tests. */
95
- function sharingHook(dir, env = process.env, now = Date.now) {
96
- const store = () => {
97
- const key = node_path_1.default.resolve(dir, state_1.EXAMPLES_FILE);
98
- let held = open.get(key);
99
- // The file may have been removed by `off`/`delete` from another process: open a fresh handle if so.
100
- if (held && !node_fs_1.default.existsSync(key)) {
101
- held.close();
102
- open.delete(key);
103
- held = undefined;
104
- }
105
- if (!held) {
106
- const made = ExampleStore.open(dir, true);
107
- if (made === null)
108
- throw new Error('the example store could not be created');
109
- open.set(key, made);
110
- held = made;
111
- }
112
- return held;
113
- };
114
- const dataOn = () => (0, state_1.effectiveLevel)(dir, env) === 'data';
115
- return {
116
- level: () => (0, state_1.effectiveLevel)(dir, env),
117
- query(queryKey, text) {
118
- try {
119
- if (dataOn())
120
- store().putQuery(queryKey, text, now());
121
- }
122
- catch {
123
- // Keeping an example must never fail the call it describes.
124
- }
125
- },
126
- memory(memoryKey, text) {
127
- try {
128
- if (dataOn())
129
- store().putMemory(memoryKey, text, now());
130
- }
131
- catch {
132
- // As above.
133
- }
134
- },
135
- };
136
- }
@@ -1,225 +0,0 @@
1
- "use strict";
2
- /**
3
- * What sharing sends, level by level. This file is the contract: the projector builds only these fields, the engine's
4
- * validator refuses anything else before a request is made, the ingestion endpoint (supabase/functions/sharing-ingest) holds
5
- * a copy of the same table and refuses what it does not list, and docs/TELEMETRY.md lists every name here (a test compares).
6
- *
7
- * Levels, in order of what leaves the machine:
8
- * off nothing is recorded for sharing and nothing is sent.
9
- * signals anonymous labelled numbers, computed inside the user's own data folder from the learning log. No text, no queries,
10
- * no memory ids, no keys, no names, no paths.
11
- * data the signals, plus the text of the examples: the query behind a retrieval, the text of the memories it returned and
12
- * of the statements that were saved. Only the user's explicit choice sends this.
13
- */
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- exports.MAX_ROWS_PER_BATCH = exports.RowError = exports.TERMINAL_REFUSALS = exports.SELF_HOSTED_BATCH_FIELDS = exports.CONSENT_FIELDS = exports.CONSENT_FIELD = exports.DELETE_FIELDS = exports.BATCH_FIELDS = exports.KINDS_WITH_CANDIDATES = exports.ROW_FIELDS = exports.CANDIDATE_FIELDS = exports.MAX_EXAMPLE_TEXT = exports.SAVE_ITEM_NAMES = exports.CANDIDATE_KIND_NAMES = exports.SAVE_TOOL_NAMES = exports.RETRIEVAL_TOOL_NAMES = exports.LENGTH_BUCKETS = exports.COUNT_BUCKETS = exports.DELAY_BUCKETS = exports.ROW_KINDS = exports.CLI_WORDING_VERSION = exports.SHARING_PROTOCOL = exports.SHARING_LEVELS = void 0;
16
- exports.levelRank = levelRank;
17
- exports.isSharingLevel = isSharingLevel;
18
- exports.lowerLevel = lowerLevel;
19
- exports.modelIdentifier = modelIdentifier;
20
- exports.fieldsAtLevel = fieldsAtLevel;
21
- exports.validateRow = validateRow;
22
- exports.SHARING_LEVELS = ['off', 'signals', 'data'];
23
- exports.SHARING_PROTOCOL = 1;
24
- /** The version of the words the CLI shows before a user picks signals or data. A dashboard sends its own wording id. */
25
- exports.CLI_WORDING_VERSION = 'engine-cli-2026-10-21';
26
- /** The order of strength: off < signals < data. */
27
- function levelRank(level) {
28
- return exports.SHARING_LEVELS.indexOf(level);
29
- }
30
- function isSharingLevel(value) {
31
- return typeof value === 'string' && exports.SHARING_LEVELS.includes(value);
32
- }
33
- /** The lower of two levels: how an override can only ever narrow what is shared. */
34
- function lowerLevel(a, b) {
35
- return levelRank(a) <= levelRank(b) ? a : b;
36
- }
37
- exports.ROW_KINDS = ['retrieval-labelled/1', 'save-outcome/1', 'session-aggregate/1'];
38
- const HOUR = '^\\d{4}-\\d{2}-\\d{2}T\\d{2}$';
39
- const LANE = '^[a-z][a-z0-9-]{0,31}$';
40
- const BUILD = '^[0-9a-f]{7,40}$';
41
- // The embedding model as `provider:model:dims` (contextRelevance.ts embeddingModelKey): a model identifier only, so no '/',
42
- // '\\', space or other path or name character can pass. A local model named by a path or `org/name` is sent as null.
43
- const MODEL = '^[a-z][a-z0-9-]{0,15}:[A-Za-z0-9._-]{1,64}:[0-9]{0,6}$';
44
- /** The model identifier a signals row may carry, or null when the recorded value is anything else. */
45
- function modelIdentifier(value) {
46
- return value !== null && new RegExp(MODEL).test(value) ? value : null;
47
- }
48
- exports.DELAY_BUCKETS = ['<1m', '<10m', '<1h', '<1d', '>=1d'];
49
- exports.COUNT_BUCKETS = ['0', '1', '2-3', '4-7', '8+'];
50
- exports.LENGTH_BUCKETS = ['<1m', '<10m', '<1h', '<6h', '>=6h'];
51
- exports.RETRIEVAL_TOOL_NAMES = ['get_context', 'search_memory', 'retrieve_memory'];
52
- exports.SAVE_TOOL_NAMES = ['remember', 'save_memory', 'remember_note'];
53
- exports.CANDIDATE_KIND_NAMES = ['fact', 'note', 'unkeyed'];
54
- exports.SAVE_ITEM_NAMES = ['fact', 'note', 'event', 'unkeyed'];
55
- /** The longest text one example may carry, in characters. A longer text is cut at this length by the projector. */
56
- exports.MAX_EXAMPLE_TEXT = 2000;
57
- const bool = { type: 'boolean' };
58
- const boolOrNull = { type: 'boolean', nullable: true };
59
- /** Fields of one entry of `candidates` in a retrieval-labelled row. */
60
- exports.CANDIDATE_FIELDS = {
61
- rank: { type: 'int' },
62
- kind: { type: 'string', oneOf: exports.CANDIDATE_KIND_NAMES },
63
- lane: { type: 'string', nullable: true, pattern: LANE },
64
- score: { type: 'number', nullable: true },
65
- matched: boolOrNull,
66
- current: boolOrNull,
67
- opened: boolOrNull,
68
- cited: boolOrNull,
69
- updated: boolOrNull,
70
- retired: boolOrNull,
71
- delay_bucket: { type: 'string', nullable: true, oneOf: exports.DELAY_BUCKETS },
72
- text: { type: 'string', nullable: true, maxLength: exports.MAX_EXAMPLE_TEXT, level: 'data' },
73
- };
74
- /** Top-level fields of each row kind. `kind` is in every row and names the row's version. */
75
- exports.ROW_FIELDS = {
76
- 'retrieval-labelled/1': {
77
- kind: { type: 'string', oneOf: ['retrieval-labelled/1'] },
78
- hour: { type: 'string', pattern: HOUR },
79
- tool: { type: 'string', oneOf: exports.RETRIEVAL_TOOL_NAMES },
80
- ranker: { type: 'string', pattern: LANE },
81
- build: { type: 'string', nullable: true, pattern: BUILD },
82
- model: { type: 'string', nullable: true, pattern: MODEL },
83
- retrieval: { type: 'string', nullable: true, oneOf: ['facts', 'hybrid', 'semantic', 'keyword'] },
84
- served: { type: 'int' },
85
- reply_tokens_bucket: { type: 'int', nullable: true },
86
- relevant: boolOrNull,
87
- truncated: boolOrNull,
88
- query: { type: 'string', nullable: true, maxLength: exports.MAX_EXAMPLE_TEXT, level: 'data' },
89
- // candidates is an array of objects; it is checked against CANDIDATE_FIELDS.
90
- },
91
- 'save-outcome/1': {
92
- kind: { type: 'string', oneOf: ['save-outcome/1'] },
93
- hour: { type: 'string', pattern: HOUR },
94
- tool: { type: 'string', oneOf: exports.SAVE_TOOL_NAMES },
95
- op: { type: 'string', oneOf: ['create', 'supersede'] },
96
- item: { type: 'string', oneOf: exports.SAVE_ITEM_NAMES },
97
- has_cites: bool,
98
- reads_bucket: { type: 'string', oneOf: exports.COUNT_BUCKETS },
99
- opened: bool,
100
- cited: bool,
101
- superseded: bool,
102
- retired: bool,
103
- text: { type: 'string', nullable: true, maxLength: exports.MAX_EXAMPLE_TEXT, level: 'data' },
104
- },
105
- 'session-aggregate/1': {
106
- kind: { type: 'string', oneOf: ['session-aggregate/1'] },
107
- hour: { type: 'string', pattern: HOUR },
108
- length_bucket: { type: 'string', oneOf: exports.LENGTH_BUCKETS },
109
- reads_bucket: { type: 'string', oneOf: exports.COUNT_BUCKETS },
110
- saves_bucket: { type: 'string', oneOf: exports.COUNT_BUCKETS },
111
- empty_reads_bucket: { type: 'string', oneOf: exports.COUNT_BUCKETS },
112
- first_call: { type: 'string', oneOf: [...exports.RETRIEVAL_TOOL_NAMES, ...exports.SAVE_TOOL_NAMES, 'other'] },
113
- },
114
- };
115
- /** Row kinds that carry a `candidates` array. */
116
- exports.KINDS_WITH_CANDIDATES = ['retrieval-labelled/1'];
117
- /** Fields of the batch envelope the engine sends (not rows). */
118
- exports.BATCH_FIELDS = ['protocol', 'action', 'batch_id', 'level', 'wording', 'rows'];
119
- exports.DELETE_FIELDS = ['protocol', 'action'];
120
- /**
121
- * A self-hosted install's batch also carries its consent record, the evidence of the choice it sends under: the version of the
122
- * record, the level chosen (at least the batch's), when it was chosen, the wording seen, and the 18-or-older confirmation.
123
- * A hosted workspace's consent is the platform's own record, so its batch has none.
124
- */
125
- exports.CONSENT_FIELD = 'consent';
126
- exports.CONSENT_FIELDS = ['version', 'level', 'changed_at', 'wording', 'adult'];
127
- exports.SELF_HOSTED_BATCH_FIELDS = [...exports.BATCH_FIELDS, exports.CONSENT_FIELD];
128
- /** The endpoint's refusals after which the same rows must never be sent again (send.ts stops instead of retrying). */
129
- exports.TERMINAL_REFUSALS = ['sharing_closed', 'consent_off', 'consent_required'];
130
- /** Every field name a level may send, for the documentation check. `signals` is a subset of `data`. */
131
- function fieldsAtLevel(level) {
132
- const names = new Set();
133
- const allowed = (spec) => spec.level === undefined || levelRank(spec.level) <= levelRank(level);
134
- for (const kind of exports.ROW_KINDS) {
135
- for (const [name, spec] of Object.entries(exports.ROW_FIELDS[kind]))
136
- if (allowed(spec))
137
- names.add(`${kind}.${name}`);
138
- }
139
- for (const [name, spec] of Object.entries(exports.CANDIDATE_FIELDS))
140
- if (allowed(spec))
141
- names.add(`retrieval-labelled/1.candidates.${name}`);
142
- return [...names].sort();
143
- }
144
- class RowError extends Error {
145
- }
146
- exports.RowError = RowError;
147
- function checkValue(path, spec, value) {
148
- if (value === null) {
149
- if (spec.nullable)
150
- return;
151
- throw new RowError(`${path} must not be null`);
152
- }
153
- switch (spec.type) {
154
- case 'string': {
155
- if (typeof value !== 'string')
156
- throw new RowError(`${path} must be a string`);
157
- if (spec.oneOf && !spec.oneOf.includes(value))
158
- throw new RowError(`${path} is not one of the listed values`);
159
- if (spec.pattern && !new RegExp(spec.pattern).test(value))
160
- throw new RowError(`${path} does not match its pattern`);
161
- if (spec.maxLength !== undefined && value.length > spec.maxLength)
162
- throw new RowError(`${path} is longer than ${spec.maxLength}`);
163
- return;
164
- }
165
- case 'int':
166
- if (typeof value !== 'number' || !Number.isInteger(value))
167
- throw new RowError(`${path} must be an integer`);
168
- return;
169
- case 'number':
170
- if (typeof value !== 'number' || !Number.isFinite(value))
171
- throw new RowError(`${path} must be a finite number`);
172
- return;
173
- case 'boolean':
174
- if (typeof value !== 'boolean')
175
- throw new RowError(`${path} must be a boolean`);
176
- return;
177
- }
178
- }
179
- function checkObject(path, specs, value, level, skip = []) {
180
- if (typeof value !== 'object' || value === null || Array.isArray(value))
181
- throw new RowError(`${path} must be an object`);
182
- const record = value;
183
- for (const name of Object.keys(record)) {
184
- if (skip.includes(name))
185
- continue;
186
- const spec = specs[name];
187
- if (spec === undefined)
188
- throw new RowError(`${path}.${name} is not a listed field`);
189
- if (spec.level !== undefined && levelRank(spec.level) > levelRank(level)) {
190
- throw new RowError(`${path}.${name} is not shared at level ${level}`);
191
- }
192
- }
193
- for (const [name, spec] of Object.entries(specs)) {
194
- if (skip.includes(name))
195
- continue;
196
- if (spec.level !== undefined && levelRank(spec.level) > levelRank(level))
197
- continue;
198
- if (spec.level !== undefined && !(name in record))
199
- continue; // a text field may be absent
200
- if (!(name in record))
201
- throw new RowError(`${path}.${name} is missing`);
202
- checkValue(`${path}.${name}`, spec, record[name]);
203
- }
204
- }
205
- /** Throws a RowError unless the row holds exactly the listed fields for its kind, with the listed value shapes, at `level`. */
206
- function validateRow(row, level) {
207
- if (typeof row !== 'object' || row === null)
208
- throw new RowError('a row must be an object');
209
- const kind = row.kind;
210
- if (typeof kind !== 'string' || !exports.ROW_KINDS.includes(kind))
211
- throw new RowError('kind is not a listed row kind');
212
- const specs = exports.ROW_FIELDS[kind];
213
- const withCandidates = exports.KINDS_WITH_CANDIDATES.includes(kind);
214
- checkObject(kind, specs, row, level, withCandidates ? ['candidates'] : []);
215
- if (withCandidates) {
216
- const candidates = row.candidates;
217
- if (!Array.isArray(candidates))
218
- throw new RowError(`${kind}.candidates must be an array`);
219
- if (candidates.length > 200)
220
- throw new RowError(`${kind}.candidates has more than 200 entries`);
221
- candidates.forEach((candidate, index) => checkObject(`${kind}.candidates[${index}]`, exports.CANDIDATE_FIELDS, candidate, level));
222
- }
223
- }
224
- /** Largest batch: rows per request. The endpoint enforces the same. */
225
- exports.MAX_ROWS_PER_BATCH = 200;
@@ -1,80 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.MIN_SHARE_INTERVAL_MS = exports.DEFAULT_SHARE_INTERVAL_MS = void 0;
4
- exports.applyInstallOff = applyInstallOff;
5
- exports.shareIntervalMs = shareIntervalMs;
6
- exports.startSharingLoop = startSharingLoop;
7
- const state_1 = require("./state");
8
- const send_1 = require("./send");
9
- /**
10
- * The install said off (`renor setup` answered No, or `renor-engine telemetry off` without a memory folder) while this workspace's
11
- * own record is still on: turn the workspace off the way its own off does, which stops collection, deletes the example texts and
12
- * owes the deletion of what was sent. effectiveLevel already reads it as off; this makes the workspace's record say so too.
13
- */
14
- async function applyInstallOff(options) {
15
- const env = options.env ?? process.env;
16
- if ((0, state_1.installLimit)(options.dir, env) !== 'off')
17
- return false;
18
- const own = (0, state_1.readConsent)(options.dir);
19
- if (own === null || own.level === 'off')
20
- return false;
21
- const install = (0, state_1.readConsent)((0, state_1.defaultSharingDir)(env));
22
- await (0, send_1.chooseLevel)({
23
- dir: options.dir, level: 'off', wording: install?.wording ?? 'install-off', adult: own.adult, source: install?.source ?? 'setup',
24
- env, fetch: options.fetch, hosted: options.hosted,
25
- });
26
- return true;
27
- }
28
- /**
29
- * The service's sender: every `intervalMs` it asks the sender to share what is final and unsent. At level off a tick reads
30
- * one small file and does nothing else. The timer never keeps the process alive and a failed tick is dropped (the rows are
31
- * still unsent, so the next tick tries again).
32
- */
33
- exports.DEFAULT_SHARE_INTERVAL_MS = 60 * 60_000;
34
- exports.MIN_SHARE_INTERVAL_MS = 60_000;
35
- function shareIntervalMs(env = process.env) {
36
- const raw = (env.RENOR_SHARING_INTERVAL_MIN ?? '').trim();
37
- if (raw === '')
38
- return exports.DEFAULT_SHARE_INTERVAL_MS;
39
- const minutes = Number(raw);
40
- if (!Number.isInteger(minutes) || minutes < 1 || minutes > 1440) {
41
- throw new Error('RENOR_SHARING_INTERVAL_MIN must be a whole number of minutes between 1 and 1440');
42
- }
43
- return minutes * 60_000;
44
- }
45
- function startSharingLoop(options) {
46
- const env = options.env ?? process.env;
47
- const intervalMs = options.intervalMs ?? shareIntervalMs(env);
48
- let running = false;
49
- const tick = async () => {
50
- if (running)
51
- return;
52
- running = true;
53
- try {
54
- if (await applyInstallOff(options))
55
- options.log?.('[sharing] off: this install chose not to share, so this workspace was turned off');
56
- await (0, send_1.settlePendingDelete)(options);
57
- if ((0, state_1.effectiveLevel)(options.dir, env) === 'off')
58
- return;
59
- const result = await (0, send_1.shareNow)(options);
60
- if (result.status === 'sent')
61
- options.log?.(`[sharing] sent ${result.rows} row(s) at level ${result.level}`);
62
- else if (result.status === 'failed')
63
- options.log?.(`[sharing] not sent: ${result.reason ?? 'unknown'}; will retry`);
64
- else if (result.status === 'paused' || result.status === 'stopped')
65
- options.log?.(`[sharing] not sent: ${result.reason ?? 'unknown'}`);
66
- }
67
- catch (err) {
68
- options.log?.(`[sharing] tick failed: ${err instanceof Error ? err.message : 'unknown'}`);
69
- }
70
- finally {
71
- running = false;
72
- }
73
- };
74
- const timer = setInterval(() => void tick(), intervalMs);
75
- timer.unref();
76
- // A tenant that sleeps when idle is woken by use, so the first tick runs soon after start instead of waiting a whole interval.
77
- const first = setTimeout(() => void tick(), options.firstTickMs ?? 30_000);
78
- first.unref();
79
- return { stop: () => { clearInterval(timer); clearTimeout(first); }, tick };
80
- }