renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,466 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.MAX_REPLY_BYTES = exports.DEFAULT_STATUS_URL = exports.DEFAULT_ENDPOINT = void 0;
7
- exports.sharingEndpoint = sharingEndpoint;
8
- exports.sharingStatusUrl = sharingStatusUrl;
9
- exports.sharingStatusOpen = sharingStatusOpen;
10
- exports.debugOn = debugOn;
11
- exports.readCapped = readCapped;
12
- exports.shareNow = shareNow;
13
- exports.deleteRemote = deleteRemote;
14
- exports.chooseLevel = chooseLevel;
15
- exports.deleteAllShared = deleteAllShared;
16
- exports.settlePendingDelete = settlePendingDelete;
17
- const node_crypto_1 = require("node:crypto");
18
- const node_fs_1 = __importDefault(require("node:fs"));
19
- const settings_1 = require("../learning/settings");
20
- const store_1 = require("../learning/store");
21
- const examples_1 = require("./examples");
22
- const fields_1 = require("./fields");
23
- const project_1 = require("./project");
24
- const state_1 = require("./state");
25
- const node_path_1 = __importDefault(require("node:path"));
26
- /**
27
- * The sender, and the changes to the choice. Everything that leaves the machine leaves through `post` below, and only from
28
- * `shareNow` (rows), `deleteRemote` (a removal request with no data in it).
29
- *
30
- * The rules, each with a test:
31
- * - level off sends nothing and writes nothing new (no file is made or opened except to read);
32
- * - signals sends the signals fields; data sends the same plus the text fields; the rows are checked against
33
- * fields.ts before a request is made, so a bug cannot widen what leaves;
34
- * - debug mode builds the exact request body and returns it without sending and without moving any watermark;
35
- * - a change of choice takes effect at the next call, in this process or another: the level is read from the record each time;
36
- * - a run reads the choice again before each request and after it; when off was chosen meanwhile, it sends nothing more and
37
- * asks the endpoint to delete what it holds (a request already on the wire may land after the deletion `off` asked for);
38
- * - at `data`, an example text is deleted once every row that can use it has been sent;
39
- * - a self-hosted batch carries the install's consent record (fields.ts CONSENT_FIELDS), which the endpoint requires;
40
- * - a refusal the endpoint means for good (fields.ts TERMINAL_REFUSALS) is never retried: the refused rows are never sent again,
41
- * `consent_off` applies off here, and `sharing_closed` or `consent_required` pause sending until the sharing status says open
42
- * again or the person chooses a level again. A deletion is never paused.
43
- */
44
- /** The endpoint the sender posts to. RENOR_SHARING_ENDPOINT replaces it (it must be https, or loopback for a test). */
45
- exports.DEFAULT_ENDPOINT = 'https://yiotrpmsxbnarzvodqyf.supabase.co/functions/v1/sharing-ingest';
46
- function sharingEndpoint(env = process.env) {
47
- const raw = (env.RENOR_SHARING_ENDPOINT ?? '').trim();
48
- if (raw === '')
49
- return exports.DEFAULT_ENDPOINT;
50
- let url;
51
- try {
52
- url = new URL(raw);
53
- }
54
- catch {
55
- throw new Error('RENOR_SHARING_ENDPOINT must be a URL');
56
- }
57
- const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname);
58
- if (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) {
59
- throw new Error('RENOR_SHARING_ENDPOINT must be https (http is allowed only for a loopback address)');
60
- }
61
- return url.toString();
62
- }
63
- /** Renor's public sharing switch (the gateway's GET /sharing/status, one bit). RENOR_SHARING_STATUS_URL replaces it. */
64
- exports.DEFAULT_STATUS_URL = 'https://mcp.renor.cloud/sharing/status';
65
- function sharingStatusUrl(env = process.env) {
66
- const raw = (env.RENOR_SHARING_STATUS_URL ?? '').trim();
67
- if (raw === '')
68
- return exports.DEFAULT_STATUS_URL;
69
- const url = new URL(raw);
70
- const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname);
71
- if (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) {
72
- throw new Error('RENOR_SHARING_STATUS_URL must be https (http is allowed only for a loopback address)');
73
- }
74
- return url.toString();
75
- }
76
- /** Whether Renor's switch is open: only an answer of exactly {"open": true} is open; any error, redirect or other answer is closed. */
77
- async function sharingStatusOpen(env = process.env) {
78
- const controller = new AbortController();
79
- const timer = setTimeout(() => controller.abort(), 15_000);
80
- try {
81
- const response = await fetch(sharingStatusUrl(env), { method: 'GET', redirect: 'manual', signal: controller.signal });
82
- if (response.status !== 200) {
83
- await response.body?.cancel().catch(() => undefined);
84
- return false;
85
- }
86
- const body = JSON.parse(await readCapped(response));
87
- return typeof body === 'object' && body !== null && JSON.stringify(body) === '{"open":true}';
88
- }
89
- catch {
90
- return false;
91
- }
92
- finally {
93
- clearTimeout(timer);
94
- }
95
- }
96
- function debugOn(env = process.env) {
97
- const raw = (env.RENOR_SHARING_DEBUG ?? '').trim().toLowerCase();
98
- return raw === '1' || raw === 'true';
99
- }
100
- /** The most of a reply body the sender reads: a refusal is a short JSON object, so anything longer is cut and the rest cancelled. */
101
- exports.MAX_REPLY_BYTES = 4096;
102
- /** Reads at most MAX_REPLY_BYTES of a reply and cancels the rest, so an endpoint (or anything in between) cannot make it read more. */
103
- async function readCapped(reply, max = exports.MAX_REPLY_BYTES) {
104
- const stream = reply.body;
105
- if (!stream)
106
- return (await reply.text()).slice(0, max);
107
- const reader = stream.getReader();
108
- const chunks = [];
109
- let size = 0;
110
- try {
111
- while (size < max) {
112
- const { done, value } = await reader.read();
113
- if (done)
114
- break;
115
- chunks.push(value);
116
- size += value.length;
117
- }
118
- }
119
- finally {
120
- await reader.cancel().catch(() => undefined);
121
- }
122
- return new TextDecoder().decode(Buffer.concat(chunks).subarray(0, max));
123
- }
124
- async function post(url, bearer, body, send, hosted = false) {
125
- const controller = new AbortController();
126
- const timer = setTimeout(() => controller.abort(), 15_000);
127
- try {
128
- const doFetch = send ?? ((u, init) => fetch(u, init));
129
- const response = await doFetch(url, {
130
- method: 'POST',
131
- headers: { 'content-type': 'application/json', authorization: `Bearer ${bearer}`, ...(hosted ? { 'x-renor-hosted': '1' } : {}) },
132
- body,
133
- signal: controller.signal,
134
- });
135
- const retry = Number(response.headers.get('retry-after'));
136
- // The refusal's code, from the endpoint's JSON body ({"ok": false, "code": ...}); null when there is none.
137
- let code = null;
138
- if (response.status >= 400) {
139
- try {
140
- const parsed = JSON.parse(await readCapped(response));
141
- if (typeof parsed === 'object' && parsed !== null && typeof parsed.code === 'string')
142
- code = parsed.code;
143
- }
144
- catch {
145
- // An unreadable body is an ordinary failure, retried.
146
- }
147
- }
148
- return { status: response.status, retryAfter: Number.isFinite(retry) && retry > 0 ? retry : null, code };
149
- }
150
- finally {
151
- clearTimeout(timer);
152
- }
153
- }
154
- /** Splits rows (oldest first) into batches of at most `max`, never between two rows with the same time, so a watermark is exact. */
155
- function chunk(rows, max) {
156
- const out = [];
157
- let index = 0;
158
- while (index < rows.length) {
159
- let end = Math.min(index + max, rows.length);
160
- while (end < rows.length && end > index + 1 && rows[end].at === rows[end - 1].at)
161
- end -= 1;
162
- out.push(rows.slice(index, end));
163
- index = end;
164
- }
165
- return out;
166
- }
167
- function batchId(state, kind, rows) {
168
- // The same chunk gets the same id on a retry, so an answer lost on the way never makes the endpoint store it twice.
169
- return (0, node_crypto_1.createHash)('sha256')
170
- .update(JSON.stringify([state.secret, kind, rows[0].at, rows[rows.length - 1].at, rows.length]))
171
- .digest('hex')
172
- .slice(0, 32);
173
- }
174
- /** Computes the rows that are final and unsent, and sends them (or, in debug mode, returns the bodies). */
175
- async function shareNow(options) {
176
- const env = options.env ?? process.env;
177
- const now = (options.now ?? Date.now)();
178
- const debug = options.debug === true || debugOn(env);
179
- const previewing = debug && options.previewLevel !== undefined;
180
- const level = previewing ? options.previewLevel : (0, state_1.effectiveLevel)(options.dir, env);
181
- const empty = (status, extra = {}) => ({ status, level, rows: 0, batches: 0, skipped: 0, ...extra });
182
- if (level === 'off')
183
- return empty('off');
184
- const record = (0, state_1.readConsent)(options.dir);
185
- const consent = record ?? (previewing ? { wording: fields_1.CLI_WORDING_VERSION } : null);
186
- if (consent === null || options.memoryDir === null)
187
- return empty('no-log', { reason: 'no learning log to read' });
188
- // A refusal for good paused sending: nothing goes out until it clears (debug only builds bodies, so it ignores the pause).
189
- const paused = debug ? undefined : (0, state_1.readState)(options.dir)?.paused;
190
- if (paused?.code === 'consent_required') {
191
- return empty('paused', { reason: 'Renor refused the consent record; nothing is sent until a level is chosen again' });
192
- }
193
- if (paused?.code === 'sharing_closed') {
194
- if (!(await (options.sharingOpen ?? (() => sharingStatusOpen(env)))())) {
195
- return empty('paused', { reason: 'Renor has closed sharing; nothing is sent until it is open again' });
196
- }
197
- const resumed = (0, state_1.readState)(options.dir);
198
- if (resumed !== null) {
199
- delete resumed.paused;
200
- (0, state_1.writeState)(options.dir, resumed);
201
- }
202
- }
203
- const file = (0, settings_1.learningDbPath)(options.memoryDir);
204
- if (!node_fs_1.default.existsSync(file))
205
- return empty('no-log', { reason: `no learning log at ${file}` });
206
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false, now: () => now });
207
- if (log === null)
208
- return empty('no-log', { reason: `no learning log at ${file}` });
209
- let exported;
210
- try {
211
- exported = log.exportLog();
212
- }
213
- finally {
214
- log.close();
215
- }
216
- // Debug reads the state if there is one and never makes it; a real send makes it on first use.
217
- const state = debug ? ((0, state_1.readState)(options.dir) ?? (0, state_1.freshState)()) : (0, state_1.ensureState)(options.dir);
218
- const examples = level === 'data' ? examples_1.ExampleStore.open(options.dir, false) : null;
219
- let projected;
220
- try {
221
- projected = (0, project_1.projectRows)(exported, { level, now, windowMs: options.windowMs, watermarks: state.watermarks, examples });
222
- }
223
- finally {
224
- examples?.close();
225
- }
226
- let skipped = 0;
227
- const valid = projected.filter(item => {
228
- try {
229
- (0, fields_1.validateRow)(item.row, level);
230
- return true;
231
- }
232
- catch {
233
- skipped += 1;
234
- return false;
235
- }
236
- });
237
- // A row that fails its own contract is never sent. Its time still passes the watermark (below), so it cannot block the rest.
238
- const byKind = new Map();
239
- for (const item of valid)
240
- byKind.set(item.kind, [...(byKind.get(item.kind) ?? []), item]);
241
- const batches = [];
242
- for (const [kind, rows] of byKind) {
243
- rows.sort((a, b) => a.at - b.at);
244
- for (const part of chunk(rows, fields_1.MAX_ROWS_PER_BATCH))
245
- batches.push({ kind, rows: part });
246
- }
247
- // Every row older than the label window is final, so it is sent once the run sends all it has; the texts only such rows use
248
- // are no longer needed. A text a later event still needs was put again at that event, which keeps it.
249
- const forgetSentExamples = () => {
250
- if (level !== 'data')
251
- return;
252
- const store = examples_1.ExampleStore.open(options.dir, false);
253
- try {
254
- store?.forgetUpTo(now - (options.windowMs ?? project_1.LABEL_WINDOW_MS));
255
- }
256
- finally {
257
- store?.close();
258
- }
259
- };
260
- if (batches.length === 0) {
261
- if (!debug)
262
- forgetSentExamples();
263
- return empty('nothing', { skipped });
264
- }
265
- // The consent record a self-hosted batch carries (the endpoint refuses one without it). A preview has no record yet: it shows
266
- // the one choosing that level would make.
267
- const evidence = {
268
- version: 1,
269
- level: record?.level ?? level,
270
- changed_at: record?.changedAt ?? new Date(now).toISOString(),
271
- wording: consent.wording,
272
- adult: record ? record.adult === true : true,
273
- };
274
- const body = (kind, rows) => ({
275
- protocol: fields_1.SHARING_PROTOCOL,
276
- action: 'batch',
277
- batch_id: batchId(state, kind, rows),
278
- level,
279
- wording: consent.wording,
280
- rows: rows.map(item => item.row),
281
- ...(options.hosted ? {} : { consent: evidence }),
282
- });
283
- if (debug) {
284
- const payloads = batches.map(batch => body(batch.kind, batch.rows));
285
- return { status: 'debug', level, rows: valid.length, batches: payloads.length, skipped, payloads };
286
- }
287
- const endpoint = sharingEndpoint(env);
288
- let sent = 0;
289
- let sentBatches = 0;
290
- const stillChosen = () => (0, fields_1.levelRank)((0, state_1.effectiveLevel)(options.dir, env)) >= (0, fields_1.levelRank)(level);
291
- const stopped = () => ({ status: 'stopped', level, rows: sent, batches: sentBatches, skipped, reason: 'the choice was lowered during the run' });
292
- for (const batch of batches) {
293
- if (!stillChosen())
294
- return stopped();
295
- // Recorded before the request, so an `off` chosen while it is on the wire knows there may be something to delete.
296
- state.everSent = true;
297
- (0, state_1.writeState)(options.dir, state);
298
- let status = null;
299
- let code = null;
300
- let failure = null;
301
- try {
302
- ({ status, code } = await post(endpoint, state.secret, JSON.stringify(body(batch.kind, batch.rows)), options.fetch, options.hosted));
303
- }
304
- catch (err) {
305
- failure = err instanceof Error ? err.message : 'request failed';
306
- }
307
- // Off was chosen while the request was out: it may have landed after the deletion off asked for, so ask again.
308
- if ((0, state_1.effectiveLevel)(options.dir, env) === 'off') {
309
- const removal = await requestDelete(options.dir, state, { env, fetch: options.fetch, hosted: options.hosted });
310
- if (removal.ok)
311
- (0, state_1.removeLocalSharingData)(options.dir);
312
- return { ...stopped(), reason: 'off was chosen during the run; the endpoint was asked to delete what it holds' };
313
- }
314
- if (failure !== null || status === null) {
315
- return { status: 'failed', level, rows: sent, batches: sentBatches, skipped, reason: failure ?? 'request failed' };
316
- }
317
- const last = batch.rows[batch.rows.length - 1].at;
318
- const key = batch.kind === 'retrieval' ? 'retrieval' : batch.kind === 'save' ? 'save' : 'session';
319
- if ((status < 200 || status >= 300) && code !== null && fields_1.TERMINAL_REFUSALS.includes(code)) {
320
- // Refused for good: these rows are never sent again, whatever happens next, and nothing more is sent now.
321
- state.watermarks[key] = Math.max(state.watermarks[key], last);
322
- if (code === 'consent_off') {
323
- (0, state_1.writeState)(options.dir, state);
324
- // The workspace's record says off: apply it here, as choosing off does (it also asks for the deletion).
325
- await chooseLevel({
326
- dir: options.dir, level: 'off', wording: consent.wording, adult: record?.adult === true, source: 'service',
327
- env, fetch: options.fetch, now: options.now, hosted: options.hosted,
328
- });
329
- return { ...stopped(), reason: 'Renor says sharing is off for this workspace; it is now off here too' };
330
- }
331
- state.paused = { code: code, at: now };
332
- (0, state_1.writeState)(options.dir, state);
333
- return {
334
- status: 'paused', level, rows: sent, batches: sentBatches, skipped,
335
- reason: code === 'sharing_closed'
336
- ? 'Renor has closed sharing; nothing is sent until it is open again'
337
- : 'Renor refused the consent record; nothing is sent until a level is chosen again',
338
- };
339
- }
340
- if (status < 200 || status >= 300) {
341
- return { status: 'failed', level, rows: sent, batches: sentBatches, skipped, reason: `the endpoint answered ${status}` };
342
- }
343
- state.watermarks[key] = Math.max(state.watermarks[key], last);
344
- state.everSent = true;
345
- (0, state_1.writeState)(options.dir, state);
346
- sent += batch.rows.length;
347
- sentBatches += 1;
348
- }
349
- // Rows that failed the contract also move the watermark, so they are not recomputed forever.
350
- if (skipped > 0) {
351
- for (const item of projected) {
352
- if (valid.includes(item))
353
- continue;
354
- const key = item.kind;
355
- state.watermarks[key] = Math.max(state.watermarks[key], item.at);
356
- }
357
- (0, state_1.writeState)(options.dir, state);
358
- }
359
- forgetSentExamples();
360
- return { status: 'sent', level, rows: sent, batches: sentBatches, skipped };
361
- }
362
- const doNotTrack = (env) => (env.DO_NOT_TRACK ?? '').trim() === '1';
363
- /**
364
- * Asks the endpoint to delete everything it holds for this contributor. A request is made only if rows were ever sent: an
365
- * install that never shared sends nothing, even to be deleted. When the endpoint cannot be reached, the removal stays owed
366
- * (deletePending) and is tried again by the next sender run.
367
- */
368
- async function deleteRemote(options) {
369
- const state = (0, state_1.readState)(options.dir);
370
- if (state === null || (!state.everSent && !state.deletePending))
371
- return { ok: true, requested: false };
372
- return requestDelete(options.dir, state, options);
373
- }
374
- /** The removal request for the contributor `state` names. A failure leaves the removal owed (deletePending) in `dir`. */
375
- async function requestDelete(dir, state, options) {
376
- const env = options.env ?? process.env;
377
- if (doNotTrack(env)) {
378
- // No request may be made from this process. The deletion is still owed: keep the secret it needs and mark it pending, so the
379
- // first run without DO_NOT_TRACK=1 sends it (settlePendingDelete).
380
- (0, state_1.writeState)(dir, { ...state, everSent: true, deletePending: true });
381
- return {
382
- ok: false, requested: false, owedUnderDoNotTrack: true,
383
- reason: 'DO_NOT_TRACK=1 forbids any request, so the deletion is recorded here and sent the first time the engine runs without it',
384
- };
385
- }
386
- try {
387
- const { status } = await post(sharingEndpoint(env), state.secret, JSON.stringify({ protocol: fields_1.SHARING_PROTOCOL, action: 'delete' }), options.fetch, options.hosted);
388
- if (status >= 200 && status < 300)
389
- return { ok: true, requested: true };
390
- // 401 on a contributor the endpoint no longer knows is the same as gone.
391
- if (status === 401 || status === 404)
392
- return { ok: true, requested: true };
393
- (0, state_1.writeState)(dir, { ...state, everSent: true, deletePending: true });
394
- return { ok: false, requested: true, reason: `the endpoint answered ${status}` };
395
- }
396
- catch (err) {
397
- (0, state_1.writeState)(dir, { ...state, everSent: true, deletePending: true });
398
- return { ok: false, requested: true, reason: err instanceof Error ? err.message : 'request failed' };
399
- }
400
- }
401
- /** Records a choice. Off with nothing ever chosen writes nothing; leaving `data` deletes the example texts at once. */
402
- async function chooseLevel(options) {
403
- const previous = (0, state_1.readConsent)(options.dir);
404
- const now = (options.now ?? Date.now)();
405
- if (options.level === 'off') {
406
- if (previous === null)
407
- return { level: 'off', changed: false, record: null, remoteDeleted: null };
408
- const record = { version: 1, level: 'off', changedAt: new Date(now).toISOString(), wording: options.wording, adult: previous.adult, source: options.source };
409
- (0, state_1.writeConsent)(options.dir, record);
410
- // Stop first: from this write on, no process records or sends anything. Then owe the deletion before discarding anything,
411
- // so a process that dies here leaves it pending (with the secret it needs) for settlePendingDelete.
412
- const owed = (0, state_1.readState)(options.dir);
413
- if (owed !== null && owed.everSent && !owed.deletePending)
414
- (0, state_1.writeState)(options.dir, { ...owed, deletePending: true });
415
- for (const name of [state_1.EXAMPLES_FILE, `${state_1.EXAMPLES_FILE}-wal`, `${state_1.EXAMPLES_FILE}-shm`])
416
- node_fs_1.default.rmSync(node_path_1.default.join(options.dir, name), { force: true });
417
- const result = await deleteRemote({ dir: options.dir, env: options.env, fetch: options.fetch, hosted: options.hosted });
418
- // The secret is removed only once the endpoint holds nothing; until then the state keeps it, with the deletion pending.
419
- if (result.ok)
420
- (0, state_1.removeLocalSharingData)(options.dir);
421
- const deleted = result.requested || result.owedUnderDoNotTrack ? result.ok : null;
422
- return {
423
- level: 'off', changed: previous.level !== 'off', record, remoteDeleted: deleted, reason: result.reason,
424
- ...(result.owedUnderDoNotTrack ? { deleteOwedUnderDoNotTrack: true } : {}),
425
- };
426
- }
427
- if (!options.adult)
428
- throw new Error('sharing needs the user to confirm they are 18 or older');
429
- // Choosing a level is the person's answer to a refusal that paused sending: sending resumes, with new rows only.
430
- const pausedState = (0, state_1.readState)(options.dir);
431
- if (pausedState?.paused !== undefined) {
432
- delete pausedState.paused;
433
- (0, state_1.writeState)(options.dir, pausedState);
434
- }
435
- if (previous !== null && previous.level === options.level && previous.adult) {
436
- return { level: options.level, changed: false, record: previous, remoteDeleted: null };
437
- }
438
- const record = { version: 1, level: options.level, changedAt: new Date(now).toISOString(), wording: options.wording, adult: true, source: options.source };
439
- (0, state_1.writeConsent)(options.dir, record);
440
- (0, state_1.ensureState)(options.dir);
441
- if ((0, fields_1.levelRank)(options.level) < (0, fields_1.levelRank)('data')) {
442
- for (const name of [state_1.EXAMPLES_FILE, `${state_1.EXAMPLES_FILE}-wal`, `${state_1.EXAMPLES_FILE}-shm`])
443
- node_fs_1.default.rmSync(node_path_1.default.join(options.dir, name), { force: true });
444
- }
445
- return { level: options.level, changed: true, record, remoteDeleted: null };
446
- }
447
- /** Deletes what the endpoint holds and what sharing keeps here (state, example texts), and keeps the choice. */
448
- async function deleteAllShared(options) {
449
- const remote = await deleteRemote(options);
450
- if (!remote.ok)
451
- return { remote, localRemoved: false };
452
- (0, state_1.removeLocalSharingData)(options.dir);
453
- return { remote, localRemoved: true };
454
- }
455
- /** The retry of a removal that could not be made earlier. Called by the loop, whatever the level. */
456
- async function settlePendingDelete(options) {
457
- const env = options.env ?? process.env;
458
- if ((0, state_1.envOverride)(env) !== null)
459
- return; // an environment override means no request at all from this process
460
- const state = (0, state_1.readState)(options.dir);
461
- if (state === null || !state.deletePending)
462
- return;
463
- const result = await deleteRemote(options);
464
- if (result.ok)
465
- (0, state_1.removeLocalSharingData)(options.dir);
466
- }