renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,466 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.MAX_REPLY_BYTES = exports.DEFAULT_STATUS_URL = exports.DEFAULT_ENDPOINT = void 0;
7
- exports.sharingEndpoint = sharingEndpoint;
8
- exports.sharingStatusUrl = sharingStatusUrl;
9
- exports.sharingStatusOpen = sharingStatusOpen;
10
- exports.debugOn = debugOn;
11
- exports.readCapped = readCapped;
12
- exports.shareNow = shareNow;
13
- exports.deleteRemote = deleteRemote;
14
- exports.chooseLevel = chooseLevel;
15
- exports.deleteAllShared = deleteAllShared;
16
- exports.settlePendingDelete = settlePendingDelete;
17
- const node_crypto_1 = require("node:crypto");
18
- const node_fs_1 = __importDefault(require("node:fs"));
19
- const settings_1 = require("../learning/settings");
20
- const store_1 = require("../learning/store");
21
- const examples_1 = require("./examples");
22
- const fields_1 = require("./fields");
23
- const project_1 = require("./project");
24
- const state_1 = require("./state");
25
- const node_path_1 = __importDefault(require("node:path"));
26
- /**
27
- * The sender, and the changes to the choice. Everything that leaves the machine leaves through `post` below, and only from
28
- * `shareNow` (rows), `deleteRemote` (a removal request with no data in it).
29
- *
30
- * The rules, each with a test:
31
- * - level off sends nothing and writes nothing new (no file is made or opened except to read);
32
- * - signals sends the signals fields; data sends the same plus the text fields; the rows are checked against
33
- * fields.ts before a request is made, so a bug cannot widen what leaves;
34
- * - debug mode builds the exact request body and returns it without sending and without moving any watermark;
35
- * - a change of choice takes effect at the next call, in this process or another: the level is read from the record each time;
36
- * - a run reads the choice again before each request and after it; when off was chosen meanwhile, it sends nothing more and
37
- * asks the endpoint to delete what it holds (a request already on the wire may land after the deletion `off` asked for);
38
- * - at `data`, an example text is deleted once every row that can use it has been sent;
39
- * - a self-hosted batch carries the install's consent record (fields.ts CONSENT_FIELDS), which the endpoint requires;
40
- * - a refusal the endpoint means for good (fields.ts TERMINAL_REFUSALS) is never retried: the refused rows are never sent again,
41
- * `consent_off` applies off here, and `sharing_closed` or `consent_required` pause sending until the sharing status says open
42
- * again or the person chooses a level again. A deletion is never paused.
43
- */
44
- /** The endpoint the sender posts to. RENOR_SHARING_ENDPOINT replaces it (it must be https, or loopback for a test). */
45
- exports.DEFAULT_ENDPOINT = 'https://yiotrpmsxbnarzvodqyf.supabase.co/functions/v1/sharing-ingest';
46
- function sharingEndpoint(env = process.env) {
47
- const raw = (env.RENOR_SHARING_ENDPOINT ?? '').trim();
48
- if (raw === '')
49
- return exports.DEFAULT_ENDPOINT;
50
- let url;
51
- try {
52
- url = new URL(raw);
53
- }
54
- catch {
55
- throw new Error('RENOR_SHARING_ENDPOINT must be a URL');
56
- }
57
- const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname);
58
- if (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) {
59
- throw new Error('RENOR_SHARING_ENDPOINT must be https (http is allowed only for a loopback address)');
60
- }
61
- return url.toString();
62
- }
63
- /** Renor's public sharing switch (the gateway's GET /sharing/status, one bit). RENOR_SHARING_STATUS_URL replaces it. */
64
- exports.DEFAULT_STATUS_URL = 'https://mcp.renor.cloud/sharing/status';
65
- function sharingStatusUrl(env = process.env) {
66
- const raw = (env.RENOR_SHARING_STATUS_URL ?? '').trim();
67
- if (raw === '')
68
- return exports.DEFAULT_STATUS_URL;
69
- const url = new URL(raw);
70
- const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname);
71
- if (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) {
72
- throw new Error('RENOR_SHARING_STATUS_URL must be https (http is allowed only for a loopback address)');
73
- }
74
- return url.toString();
75
- }
76
- /** Whether Renor's switch is open: only an answer of exactly {"open": true} is open; any error, redirect or other answer is closed. */
77
- async function sharingStatusOpen(env = process.env) {
78
- const controller = new AbortController();
79
- const timer = setTimeout(() => controller.abort(), 15_000);
80
- try {
81
- const response = await fetch(sharingStatusUrl(env), { method: 'GET', redirect: 'manual', signal: controller.signal });
82
- if (response.status !== 200) {
83
- await response.body?.cancel().catch(() => undefined);
84
- return false;
85
- }
86
- const body = JSON.parse(await readCapped(response));
87
- return typeof body === 'object' && body !== null && JSON.stringify(body) === '{"open":true}';
88
- }
89
- catch {
90
- return false;
91
- }
92
- finally {
93
- clearTimeout(timer);
94
- }
95
- }
96
- function debugOn(env = process.env) {
97
- const raw = (env.RENOR_SHARING_DEBUG ?? '').trim().toLowerCase();
98
- return raw === '1' || raw === 'true';
99
- }
100
- /** The most of a reply body the sender reads: a refusal is a short JSON object, so anything longer is cut and the rest cancelled. */
101
- exports.MAX_REPLY_BYTES = 4096;
102
- /** Reads at most MAX_REPLY_BYTES of a reply and cancels the rest, so an endpoint (or anything in between) cannot make it read more. */
103
- async function readCapped(reply, max = exports.MAX_REPLY_BYTES) {
104
- const stream = reply.body;
105
- if (!stream)
106
- return (await reply.text()).slice(0, max);
107
- const reader = stream.getReader();
108
- const chunks = [];
109
- let size = 0;
110
- try {
111
- while (size < max) {
112
- const { done, value } = await reader.read();
113
- if (done)
114
- break;
115
- chunks.push(value);
116
- size += value.length;
117
- }
118
- }
119
- finally {
120
- await reader.cancel().catch(() => undefined);
121
- }
122
- return new TextDecoder().decode(Buffer.concat(chunks).subarray(0, max));
123
- }
124
- async function post(url, bearer, body, send, hosted = false) {
125
- const controller = new AbortController();
126
- const timer = setTimeout(() => controller.abort(), 15_000);
127
- try {
128
- const doFetch = send ?? ((u, init) => fetch(u, init));
129
- const response = await doFetch(url, {
130
- method: 'POST',
131
- headers: { 'content-type': 'application/json', authorization: `Bearer ${bearer}`, ...(hosted ? { 'x-renor-hosted': '1' } : {}) },
132
- body,
133
- signal: controller.signal,
134
- });
135
- const retry = Number(response.headers.get('retry-after'));
136
- // The refusal's code, from the endpoint's JSON body ({"ok": false, "code": ...}); null when there is none.
137
- let code = null;
138
- if (response.status >= 400) {
139
- try {
140
- const parsed = JSON.parse(await readCapped(response));
141
- if (typeof parsed === 'object' && parsed !== null && typeof parsed.code === 'string')
142
- code = parsed.code;
143
- }
144
- catch {
145
- // An unreadable body is an ordinary failure, retried.
146
- }
147
- }
148
- return { status: response.status, retryAfter: Number.isFinite(retry) && retry > 0 ? retry : null, code };
149
- }
150
- finally {
151
- clearTimeout(timer);
152
- }
153
- }
154
- /** Splits rows (oldest first) into batches of at most `max`, never between two rows with the same time, so a watermark is exact. */
155
- function chunk(rows, max) {
156
- const out = [];
157
- let index = 0;
158
- while (index < rows.length) {
159
- let end = Math.min(index + max, rows.length);
160
- while (end < rows.length && end > index + 1 && rows[end].at === rows[end - 1].at)
161
- end -= 1;
162
- out.push(rows.slice(index, end));
163
- index = end;
164
- }
165
- return out;
166
- }
167
- function batchId(state, kind, rows) {
168
- // The same chunk gets the same id on a retry, so an answer lost on the way never makes the endpoint store it twice.
169
- return (0, node_crypto_1.createHash)('sha256')
170
- .update(JSON.stringify([state.secret, kind, rows[0].at, rows[rows.length - 1].at, rows.length]))
171
- .digest('hex')
172
- .slice(0, 32);
173
- }
174
- /** Computes the rows that are final and unsent, and sends them (or, in debug mode, returns the bodies). */
175
- async function shareNow(options) {
176
- const env = options.env ?? process.env;
177
- const now = (options.now ?? Date.now)();
178
- const debug = options.debug === true || debugOn(env);
179
- const previewing = debug && options.previewLevel !== undefined;
180
- const level = previewing ? options.previewLevel : (0, state_1.effectiveLevel)(options.dir, env);
181
- const empty = (status, extra = {}) => ({ status, level, rows: 0, batches: 0, skipped: 0, ...extra });
182
- if (level === 'off')
183
- return empty('off');
184
- const record = (0, state_1.readConsent)(options.dir);
185
- const consent = record ?? (previewing ? { wording: fields_1.CLI_WORDING_VERSION } : null);
186
- if (consent === null || options.memoryDir === null)
187
- return empty('no-log', { reason: 'no learning log to read' });
188
- // A refusal for good paused sending: nothing goes out until it clears (debug only builds bodies, so it ignores the pause).
189
- const paused = debug ? undefined : (0, state_1.readState)(options.dir)?.paused;
190
- if (paused?.code === 'consent_required') {
191
- return empty('paused', { reason: 'Renor refused the consent record; nothing is sent until a level is chosen again' });
192
- }
193
- if (paused?.code === 'sharing_closed') {
194
- if (!(await (options.sharingOpen ?? (() => sharingStatusOpen(env)))())) {
195
- return empty('paused', { reason: 'Renor has closed sharing; nothing is sent until it is open again' });
196
- }
197
- const resumed = (0, state_1.readState)(options.dir);
198
- if (resumed !== null) {
199
- delete resumed.paused;
200
- (0, state_1.writeState)(options.dir, resumed);
201
- }
202
- }
203
- const file = (0, settings_1.learningDbPath)(options.memoryDir);
204
- if (!node_fs_1.default.existsSync(file))
205
- return empty('no-log', { reason: `no learning log at ${file}` });
206
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false, now: () => now });
207
- if (log === null)
208
- return empty('no-log', { reason: `no learning log at ${file}` });
209
- let exported;
210
- try {
211
- exported = log.exportLog();
212
- }
213
- finally {
214
- log.close();
215
- }
216
- // Debug reads the state if there is one and never makes it; a real send makes it on first use.
217
- const state = debug ? ((0, state_1.readState)(options.dir) ?? (0, state_1.freshState)()) : (0, state_1.ensureState)(options.dir);
218
- const examples = level === 'data' ? examples_1.ExampleStore.open(options.dir, false) : null;
219
- let projected;
220
- try {
221
- projected = (0, project_1.projectRows)(exported, { level, now, windowMs: options.windowMs, watermarks: state.watermarks, examples });
222
- }
223
- finally {
224
- examples?.close();
225
- }
226
- let skipped = 0;
227
- const valid = projected.filter(item => {
228
- try {
229
- (0, fields_1.validateRow)(item.row, level);
230
- return true;
231
- }
232
- catch {
233
- skipped += 1;
234
- return false;
235
- }
236
- });
237
- // A row that fails its own contract is never sent. Its time still passes the watermark (below), so it cannot block the rest.
238
- const byKind = new Map();
239
- for (const item of valid)
240
- byKind.set(item.kind, [...(byKind.get(item.kind) ?? []), item]);
241
- const batches = [];
242
- for (const [kind, rows] of byKind) {
243
- rows.sort((a, b) => a.at - b.at);
244
- for (const part of chunk(rows, fields_1.MAX_ROWS_PER_BATCH))
245
- batches.push({ kind, rows: part });
246
- }
247
- // Every row older than the label window is final, so it is sent once the run sends all it has; the texts only such rows use
248
- // are no longer needed. A text a later event still needs was put again at that event, which keeps it.
249
- const forgetSentExamples = () => {
250
- if (level !== 'data')
251
- return;
252
- const store = examples_1.ExampleStore.open(options.dir, false);
253
- try {
254
- store?.forgetUpTo(now - (options.windowMs ?? project_1.LABEL_WINDOW_MS));
255
- }
256
- finally {
257
- store?.close();
258
- }
259
- };
260
- if (batches.length === 0) {
261
- if (!debug)
262
- forgetSentExamples();
263
- return empty('nothing', { skipped });
264
- }
265
- // The consent record a self-hosted batch carries (the endpoint refuses one without it). A preview has no record yet: it shows
266
- // the one choosing that level would make.
267
- const evidence = {
268
- version: 1,
269
- level: record?.level ?? level,
270
- changed_at: record?.changedAt ?? new Date(now).toISOString(),
271
- wording: consent.wording,
272
- adult: record ? record.adult === true : true,
273
- };
274
- const body = (kind, rows) => ({
275
- protocol: fields_1.SHARING_PROTOCOL,
276
- action: 'batch',
277
- batch_id: batchId(state, kind, rows),
278
- level,
279
- wording: consent.wording,
280
- rows: rows.map(item => item.row),
281
- ...(options.hosted ? {} : { consent: evidence }),
282
- });
283
- if (debug) {
284
- const payloads = batches.map(batch => body(batch.kind, batch.rows));
285
- return { status: 'debug', level, rows: valid.length, batches: payloads.length, skipped, payloads };
286
- }
287
- const endpoint = sharingEndpoint(env);
288
- let sent = 0;
289
- let sentBatches = 0;
290
- const stillChosen = () => (0, fields_1.levelRank)((0, state_1.effectiveLevel)(options.dir, env)) >= (0, fields_1.levelRank)(level);
291
- const stopped = () => ({ status: 'stopped', level, rows: sent, batches: sentBatches, skipped, reason: 'the choice was lowered during the run' });
292
- for (const batch of batches) {
293
- if (!stillChosen())
294
- return stopped();
295
- // Recorded before the request, so an `off` chosen while it is on the wire knows there may be something to delete.
296
- state.everSent = true;
297
- (0, state_1.writeState)(options.dir, state);
298
- let status = null;
299
- let code = null;
300
- let failure = null;
301
- try {
302
- ({ status, code } = await post(endpoint, state.secret, JSON.stringify(body(batch.kind, batch.rows)), options.fetch, options.hosted));
303
- }
304
- catch (err) {
305
- failure = err instanceof Error ? err.message : 'request failed';
306
- }
307
- // Off was chosen while the request was out: it may have landed after the deletion off asked for, so ask again.
308
- if ((0, state_1.effectiveLevel)(options.dir, env) === 'off') {
309
- const removal = await requestDelete(options.dir, state, { env, fetch: options.fetch, hosted: options.hosted });
310
- if (removal.ok)
311
- (0, state_1.removeLocalSharingData)(options.dir);
312
- return { ...stopped(), reason: 'off was chosen during the run; the endpoint was asked to delete what it holds' };
313
- }
314
- if (failure !== null || status === null) {
315
- return { status: 'failed', level, rows: sent, batches: sentBatches, skipped, reason: failure ?? 'request failed' };
316
- }
317
- const last = batch.rows[batch.rows.length - 1].at;
318
- const key = batch.kind === 'retrieval' ? 'retrieval' : batch.kind === 'save' ? 'save' : 'session';
319
- if ((status < 200 || status >= 300) && code !== null && fields_1.TERMINAL_REFUSALS.includes(code)) {
320
- // Refused for good: these rows are never sent again, whatever happens next, and nothing more is sent now.
321
- state.watermarks[key] = Math.max(state.watermarks[key], last);
322
- if (code === 'consent_off') {
323
- (0, state_1.writeState)(options.dir, state);
324
- // The workspace's record says off: apply it here, as choosing off does (it also asks for the deletion).
325
- await chooseLevel({
326
- dir: options.dir, level: 'off', wording: consent.wording, adult: record?.adult === true, source: 'service',
327
- env, fetch: options.fetch, now: options.now, hosted: options.hosted,
328
- });
329
- return { ...stopped(), reason: 'Renor says sharing is off for this workspace; it is now off here too' };
330
- }
331
- state.paused = { code: code, at: now };
332
- (0, state_1.writeState)(options.dir, state);
333
- return {
334
- status: 'paused', level, rows: sent, batches: sentBatches, skipped,
335
- reason: code === 'sharing_closed'
336
- ? 'Renor has closed sharing; nothing is sent until it is open again'
337
- : 'Renor refused the consent record; nothing is sent until a level is chosen again',
338
- };
339
- }
340
- if (status < 200 || status >= 300) {
341
- return { status: 'failed', level, rows: sent, batches: sentBatches, skipped, reason: `the endpoint answered ${status}` };
342
- }
343
- state.watermarks[key] = Math.max(state.watermarks[key], last);
344
- state.everSent = true;
345
- (0, state_1.writeState)(options.dir, state);
346
- sent += batch.rows.length;
347
- sentBatches += 1;
348
- }
349
- // Rows that failed the contract also move the watermark, so they are not recomputed forever.
350
- if (skipped > 0) {
351
- for (const item of projected) {
352
- if (valid.includes(item))
353
- continue;
354
- const key = item.kind;
355
- state.watermarks[key] = Math.max(state.watermarks[key], item.at);
356
- }
357
- (0, state_1.writeState)(options.dir, state);
358
- }
359
- forgetSentExamples();
360
- return { status: 'sent', level, rows: sent, batches: sentBatches, skipped };
361
- }
362
- const doNotTrack = (env) => (env.DO_NOT_TRACK ?? '').trim() === '1';
363
- /**
364
- * Asks the endpoint to delete everything it holds for this contributor. A request is made only if rows were ever sent: an
365
- * install that never shared sends nothing, even to be deleted. When the endpoint cannot be reached, the removal stays owed
366
- * (deletePending) and is tried again by the next sender run.
367
- */
368
- async function deleteRemote(options) {
369
- const state = (0, state_1.readState)(options.dir);
370
- if (state === null || (!state.everSent && !state.deletePending))
371
- return { ok: true, requested: false };
372
- return requestDelete(options.dir, state, options);
373
- }
374
- /** The removal request for the contributor `state` names. A failure leaves the removal owed (deletePending) in `dir`. */
375
- async function requestDelete(dir, state, options) {
376
- const env = options.env ?? process.env;
377
- if (doNotTrack(env)) {
378
- // No request may be made from this process. The deletion is still owed: keep the secret it needs and mark it pending, so the
379
- // first run without DO_NOT_TRACK=1 sends it (settlePendingDelete).
380
- (0, state_1.writeState)(dir, { ...state, everSent: true, deletePending: true });
381
- return {
382
- ok: false, requested: false, owedUnderDoNotTrack: true,
383
- reason: 'DO_NOT_TRACK=1 forbids any request, so the deletion is recorded here and sent the first time the engine runs without it',
384
- };
385
- }
386
- try {
387
- const { status } = await post(sharingEndpoint(env), state.secret, JSON.stringify({ protocol: fields_1.SHARING_PROTOCOL, action: 'delete' }), options.fetch, options.hosted);
388
- if (status >= 200 && status < 300)
389
- return { ok: true, requested: true };
390
- // 401 on a contributor the endpoint no longer knows is the same as gone.
391
- if (status === 401 || status === 404)
392
- return { ok: true, requested: true };
393
- (0, state_1.writeState)(dir, { ...state, everSent: true, deletePending: true });
394
- return { ok: false, requested: true, reason: `the endpoint answered ${status}` };
395
- }
396
- catch (err) {
397
- (0, state_1.writeState)(dir, { ...state, everSent: true, deletePending: true });
398
- return { ok: false, requested: true, reason: err instanceof Error ? err.message : 'request failed' };
399
- }
400
- }
401
- /** Records a choice. Off with nothing ever chosen writes nothing; leaving `data` deletes the example texts at once. */
402
- async function chooseLevel(options) {
403
- const previous = (0, state_1.readConsent)(options.dir);
404
- const now = (options.now ?? Date.now)();
405
- if (options.level === 'off') {
406
- if (previous === null)
407
- return { level: 'off', changed: false, record: null, remoteDeleted: null };
408
- const record = { version: 1, level: 'off', changedAt: new Date(now).toISOString(), wording: options.wording, adult: previous.adult, source: options.source };
409
- (0, state_1.writeConsent)(options.dir, record);
410
- // Stop first: from this write on, no process records or sends anything. Then owe the deletion before discarding anything,
411
- // so a process that dies here leaves it pending (with the secret it needs) for settlePendingDelete.
412
- const owed = (0, state_1.readState)(options.dir);
413
- if (owed !== null && owed.everSent && !owed.deletePending)
414
- (0, state_1.writeState)(options.dir, { ...owed, deletePending: true });
415
- for (const name of [state_1.EXAMPLES_FILE, `${state_1.EXAMPLES_FILE}-wal`, `${state_1.EXAMPLES_FILE}-shm`])
416
- node_fs_1.default.rmSync(node_path_1.default.join(options.dir, name), { force: true });
417
- const result = await deleteRemote({ dir: options.dir, env: options.env, fetch: options.fetch, hosted: options.hosted });
418
- // The secret is removed only once the endpoint holds nothing; until then the state keeps it, with the deletion pending.
419
- if (result.ok)
420
- (0, state_1.removeLocalSharingData)(options.dir);
421
- const deleted = result.requested || result.owedUnderDoNotTrack ? result.ok : null;
422
- return {
423
- level: 'off', changed: previous.level !== 'off', record, remoteDeleted: deleted, reason: result.reason,
424
- ...(result.owedUnderDoNotTrack ? { deleteOwedUnderDoNotTrack: true } : {}),
425
- };
426
- }
427
- if (!options.adult)
428
- throw new Error('sharing needs the user to confirm they are 18 or older');
429
- // Choosing a level is the person's answer to a refusal that paused sending: sending resumes, with new rows only.
430
- const pausedState = (0, state_1.readState)(options.dir);
431
- if (pausedState?.paused !== undefined) {
432
- delete pausedState.paused;
433
- (0, state_1.writeState)(options.dir, pausedState);
434
- }
435
- if (previous !== null && previous.level === options.level && previous.adult) {
436
- return { level: options.level, changed: false, record: previous, remoteDeleted: null };
437
- }
438
- const record = { version: 1, level: options.level, changedAt: new Date(now).toISOString(), wording: options.wording, adult: true, source: options.source };
439
- (0, state_1.writeConsent)(options.dir, record);
440
- (0, state_1.ensureState)(options.dir);
441
- if ((0, fields_1.levelRank)(options.level) < (0, fields_1.levelRank)('data')) {
442
- for (const name of [state_1.EXAMPLES_FILE, `${state_1.EXAMPLES_FILE}-wal`, `${state_1.EXAMPLES_FILE}-shm`])
443
- node_fs_1.default.rmSync(node_path_1.default.join(options.dir, name), { force: true });
444
- }
445
- return { level: options.level, changed: true, record, remoteDeleted: null };
446
- }
447
- /** Deletes what the endpoint holds and what sharing keeps here (state, example texts), and keeps the choice. */
448
- async function deleteAllShared(options) {
449
- const remote = await deleteRemote(options);
450
- if (!remote.ok)
451
- return { remote, localRemoved: false };
452
- (0, state_1.removeLocalSharingData)(options.dir);
453
- return { remote, localRemoved: true };
454
- }
455
- /** The retry of a removal that could not be made earlier. Called by the loop, whatever the level. */
456
- async function settlePendingDelete(options) {
457
- const env = options.env ?? process.env;
458
- if ((0, state_1.envOverride)(env) !== null)
459
- return; // an environment override means no request at all from this process
460
- const state = (0, state_1.readState)(options.dir);
461
- if (state === null || !state.deletePending)
462
- return;
463
- const result = await deleteRemote(options);
464
- if (result.ok)
465
- (0, state_1.removeLocalSharingData)(options.dir);
466
- }