tensorlake 0.5.143 → 0.5.145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +11 -159
  2. package/applications.cjs +3 -0
  3. package/applications.d.cts +1 -0
  4. package/applications.d.ts +1 -0
  5. package/applications.js +1 -0
  6. package/index.cjs +3 -0
  7. package/index.d.cts +1 -0
  8. package/index.d.ts +1 -0
  9. package/index.js +1 -0
  10. package/package.json +25 -68
  11. package/bin/function-executor.js +0 -8
  12. package/bin/tensorlake-create-sandbox-image.js +0 -9
  13. package/bin/tensorlake-deploy.cjs +0 -8
  14. package/bin/tensorlake-import-sandbox-image.js +0 -9
  15. package/bin/tensorlake-typescript-function-runner.js +0 -8
  16. package/dist/applications/index.cjs +0 -4821
  17. package/dist/applications/index.cjs.map +0 -1
  18. package/dist/applications/index.d.cts +0 -130
  19. package/dist/applications/index.d.ts +0 -130
  20. package/dist/applications/index.js +0 -4782
  21. package/dist/applications/index.js.map +0 -1
  22. package/dist/function-agent/main.cjs +0 -1838
  23. package/dist/function-agent/main.cjs.map +0 -1
  24. package/dist/function-agent/main.d.cts +0 -3
  25. package/dist/function-agent/main.d.ts +0 -3
  26. package/dist/function-agent/main.js +0 -1825
  27. package/dist/function-agent/main.js.map +0 -1
  28. package/dist/function-executor/main.cjs +0 -4944
  29. package/dist/function-executor/main.cjs.map +0 -1
  30. package/dist/function-executor/main.d.cts +0 -4
  31. package/dist/function-executor/main.d.ts +0 -4
  32. package/dist/function-executor/main.js +0 -4902
  33. package/dist/function-executor/main.js.map +0 -1
  34. package/dist/index.cjs +0 -10184
  35. package/dist/index.cjs.map +0 -1
  36. package/dist/index.d.cts +0 -2431
  37. package/dist/index.d.ts +0 -2431
  38. package/dist/index.js +0 -10108
  39. package/dist/index.js.map +0 -1
  40. package/dist/native-worker.cjs +0 -278
  41. package/dist/native-worker.cjs.map +0 -1
  42. package/dist/native-worker.d.cts +0 -2
  43. package/dist/native-worker.d.ts +0 -2
  44. package/dist/native-worker.js +0 -276
  45. package/dist/native-worker.js.map +0 -1
  46. package/dist/remote-CNJ9TmcQ.d.cts +0 -283
  47. package/dist/remote-dblFxKug.d.ts +0 -283
  48. package/dist/sandbox-image-ChzWDeZy.d.cts +0 -357
  49. package/dist/sandbox-image-ChzWDeZy.d.ts +0 -357
  50. package/dist/sandbox-image.cjs +0 -1992
  51. package/dist/sandbox-image.cjs.map +0 -1
  52. package/dist/sandbox-image.d.cts +0 -1
  53. package/dist/sandbox-image.d.ts +0 -1
  54. package/dist/sandbox-image.js +0 -1987
  55. package/dist/sandbox-image.js.map +0 -1
  56. package/lib/libc.cjs +0 -61
  57. package/lib/libc.d.cts +0 -1
  58. package/lib/runtime.cjs +0 -155
  59. package/runtime/function-executor/manifest.json +0 -59
  60. package/runtime/function-executor/package/bin/function-executor.js +0 -8
  61. package/runtime/function-executor/package/dist/function-executor/main.js +0 -4902
  62. package/runtime/function-executor/package/dist/function-executor/main.js.map +0 -1
  63. package/runtime/function-executor/package/dist/native-worker.cjs +0 -278
  64. package/runtime/function-executor/package/dist/native-worker.cjs.map +0 -1
  65. package/runtime/function-executor/package/npm-shrinkwrap.json +0 -475
  66. package/runtime/function-executor/package/package.json +0 -22
  67. package/runtime/function-executor/package/proto/README.md +0 -18
  68. package/runtime/function-executor/package/proto/tensorlake/function_executor/proto/function_executor.proto +0 -613
  69. package/runtime/function-executor/package/proto/tensorlake/function_executor/proto/status.proto +0 -42
  70. package/runtime/typescript-function-runner/manifest.json +0 -44
  71. package/runtime/typescript-function-runner/package/bin/tensorlake-typescript-function-runner.js +0 -8
  72. package/runtime/typescript-function-runner/package/dist/function-agent/main.js +0 -1825
  73. package/runtime/typescript-function-runner/package/dist/function-agent/main.js.map +0 -1
  74. package/runtime/typescript-function-runner/package/dist/native-worker.cjs +0 -278
  75. package/runtime/typescript-function-runner/package/dist/native-worker.cjs.map +0 -1
  76. package/runtime/typescript-function-runner/package/npm-shrinkwrap.json +0 -580
  77. package/runtime/typescript-function-runner/package/package.json +0 -25
@@ -1,4944 +0,0 @@
1
- "use strict";
2
- var __create = Object.create;
3
- var __defProp = Object.defineProperty;
4
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
- var __hasOwnProp = Object.prototype.hasOwnProperty;
8
- var __export = (target, all) => {
9
- for (var name in all)
10
- __defProp(target, name, { get: all[name], enumerable: true });
11
- };
12
- var __copyProps = (to, from, except, desc) => {
13
- if (from && typeof from === "object" || typeof from === "function") {
14
- for (let key of __getOwnPropNames(from))
15
- if (!__hasOwnProp.call(to, key) && key !== except)
16
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
- }
18
- return to;
19
- };
20
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
- // If the importer is in node compatibility mode or this is not an ESM
22
- // file that has been converted to a CommonJS file using a Babel-
23
- // compatible transform (i.e. "__esModule" has not been set), then set
24
- // "default" to the CommonJS "module.exports" for node compatibility.
25
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
- mod
27
- ));
28
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
-
30
- // src/function-executor/main.ts
31
- var main_exports = {};
32
- __export(main_exports, {
33
- functionExecutorBindAddress: () => functionExecutorBindAddress,
34
- main: () => main
35
- });
36
- module.exports = __toCommonJS(main_exports);
37
-
38
- // node_modules/tsup/assets/cjs_shims.js
39
- var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.tagName.toUpperCase() === "SCRIPT" ? document.currentScript.src : new URL("main.js", document.baseURI).href;
40
- var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
41
-
42
- // src/function-executor/main.ts
43
- var import_node_fs = require("fs");
44
-
45
- // src/native-worker-client.ts
46
- var import_node_module = require("module");
47
- var import_node_worker_threads = require("worker_threads");
48
-
49
- // src/native-worker-protocol.ts
50
- var cancelNativeCall = /* @__PURE__ */ Symbol.for("tensorlake.native.cancel.v1");
51
- var closeNativeHandle = /* @__PURE__ */ Symbol.for("tensorlake.native.close.v1");
52
-
53
- // src/defaults.ts
54
- var SDK_VERSION = "0.5.143";
55
- var API_URL = process.env.TENSORLAKE_API_URL ?? "https://api.tensorlake.ai";
56
- var API_KEY = process.env.TENSORLAKE_API_KEY ?? void 0;
57
- var NAMESPACE = process.env.INDEXIFY_NAMESPACE ?? "default";
58
- var SANDBOX_PROXY_URL = process.env.TENSORLAKE_SANDBOX_PROXY_URL ?? "https://sandbox.tensorlake.ai";
59
-
60
- // src/url.ts
61
- function trimTrailingSlashes(url) {
62
- return url.endsWith("/") ? url.slice(0, -1) : url;
63
- }
64
- function resolveProxyTarget(proxyUrl, sandboxId) {
65
- try {
66
- const parsed = new URL(proxyUrl);
67
- const host = parsed.hostname;
68
- if (host === "localhost" || host === "127.0.0.1") {
69
- return {
70
- baseUrl: trimTrailingSlashes(proxyUrl),
71
- hostHeader: `${sandboxId}.local`,
72
- sandboxIdHeader: void 0
73
- };
74
- }
75
- const port = parsed.port ? `:${parsed.port}` : "";
76
- return {
77
- baseUrl: `${parsed.protocol}//${host}${port}`,
78
- hostHeader: void 0,
79
- sandboxIdHeader: sandboxId
80
- };
81
- } catch {
82
- return {
83
- baseUrl: `${trimTrailingSlashes(proxyUrl)}/${sandboxId}`,
84
- hostHeader: void 0,
85
- sandboxIdHeader: void 0
86
- };
87
- }
88
- }
89
-
90
- // src/native-worker-client.ts
91
- var NativeWorkerClient = class {
92
- constructor(workerPath = () => (0, import_node_module.createRequire)(importMetaUrl).resolve(
93
- "tensorlake/internal/native-worker"
94
- )) {
95
- this.workerPath = workerPath;
96
- }
97
- worker;
98
- nextId = 1;
99
- registered = /* @__PURE__ */ new Set();
100
- pending = /* @__PURE__ */ new Map();
101
- finalizers = new FinalizationRegistry(
102
- (id) => this.release(id)
103
- );
104
- handle(kind, args, parent) {
105
- const handle = { id: this.nextId++, kind, args, parent };
106
- this.finalizers.register(handle, handle.id);
107
- let closed = false;
108
- return new Proxy(
109
- {},
110
- {
111
- get: (_target, method) => {
112
- if (method === "then") return void 0;
113
- if (method === closeNativeHandle)
114
- return () => {
115
- closed = true;
116
- this.release(handle.id);
117
- };
118
- if (method === "connectProxy")
119
- return (...proxyArgs) => {
120
- if (closed) throw new Error("Tensorlake native client is closed");
121
- return this.handle("NativeSandboxProxyClient", proxyArgs, handle);
122
- };
123
- if (method === "baseUrl")
124
- return () => resolveProxyTarget(String(args[0]), String(args[1])).baseUrl;
125
- if (typeof method !== "string") return void 0;
126
- return (...callArgs) => {
127
- if (closed)
128
- return Promise.reject(
129
- new Error("Tensorlake native client is closed")
130
- );
131
- const emit = typeof callArgs.at(-1) === "function" ? callArgs.pop() : void 0;
132
- return this.call(handle, method, callArgs, emit);
133
- };
134
- }
135
- }
136
- );
137
- }
138
- call(handle, method, args, emit) {
139
- const id = this.nextId++;
140
- let worker;
141
- const promise = new Promise((resolve, reject) => {
142
- try {
143
- worker = this.getWorker();
144
- this.pending.set(id, { resolve, reject, emit, handle });
145
- worker.ref();
146
- if (handle) this.register(handle, worker);
147
- this.send(worker, {
148
- type: "call",
149
- id,
150
- handleId: handle?.id,
151
- method,
152
- args,
153
- stream: !!emit
154
- });
155
- } catch (error) {
156
- this.pending.delete(id);
157
- this.unrefIfIdle();
158
- reject(error);
159
- }
160
- });
161
- promise[cancelNativeCall] = () => {
162
- const pending = this.pending.get(id);
163
- if (pending && worker) {
164
- pending.emit = void 0;
165
- this.send(worker, { type: "cancel", id });
166
- }
167
- };
168
- return promise;
169
- }
170
- getWorker() {
171
- if (this.worker) return this.worker;
172
- const worker = new import_node_worker_threads.Worker(this.workerPath(), {
173
- // The bundled worker needs no application loaders or telemetry preloads.
174
- // NODE_OPTIONS is parsed independently of execArgv by Node.
175
- execArgv: [],
176
- env: { ...process.env, NODE_OPTIONS: "" },
177
- name: "tensorlake-native"
178
- });
179
- this.worker = worker;
180
- worker.on("message", (message) => {
181
- const pending = this.pending.get(message.id);
182
- if (!pending || this.worker !== worker) return;
183
- if (message.type === "event") {
184
- Promise.resolve().then(() => pending.emit?.(message.value)).then(
185
- () => {
186
- if (this.worker === worker)
187
- this.send(worker, { type: "ack", id: message.id });
188
- },
189
- (error) => {
190
- this.send(worker, { type: "cancel", id: message.id });
191
- pending.reject(error);
192
- }
193
- );
194
- return;
195
- }
196
- this.pending.delete(message.id);
197
- if (message.type === "error") {
198
- const error = new Error(message.error.message);
199
- error.name = message.error.name;
200
- error.stack = message.error.stack ?? error.stack;
201
- pending.reject(error);
202
- } else {
203
- pending.resolve(message.value);
204
- }
205
- this.unrefIfIdle();
206
- });
207
- const fail = (error) => {
208
- if (this.worker !== worker) return;
209
- this.worker = void 0;
210
- this.registered.clear();
211
- for (const pending of this.pending.values()) pending.reject(error);
212
- this.pending.clear();
213
- void worker.terminate();
214
- };
215
- worker.on("error", fail);
216
- worker.on("messageerror", fail);
217
- worker.on(
218
- "exit",
219
- (code) => fail(
220
- new Error(
221
- `Tensorlake native worker exited (${code}); in-flight operations were not retried`
222
- )
223
- )
224
- );
225
- worker.unref();
226
- return worker;
227
- }
228
- register(handle, worker) {
229
- if (this.registered.has(handle.id)) return;
230
- if (handle.parent) this.register(handle.parent, worker);
231
- this.send(worker, {
232
- type: "create",
233
- handle: { id: handle.id, kind: handle.kind, args: handle.args },
234
- parentId: handle.parent?.id
235
- });
236
- this.registered.add(handle.id);
237
- }
238
- release(handleId) {
239
- if (this.worker && this.registered.delete(handleId))
240
- this.send(this.worker, { type: "release", handleId });
241
- }
242
- send(worker, message) {
243
- worker.postMessage(message);
244
- }
245
- unrefIfIdle() {
246
- if (this.pending.size === 0) this.worker?.unref();
247
- }
248
- };
249
- var runtimeKey = /* @__PURE__ */ Symbol.for(
250
- `tensorlake.native.worker.${false ? "source" : "0.5.143"}.v1`
251
- );
252
- var runtimes = globalThis;
253
- function configureNativeWorker(workerPath) {
254
- runtimes[runtimeKey] ??= new NativeWorkerClient(workerPath);
255
- }
256
-
257
- // src/function-executor/main.ts
258
- var import_node_path2 = __toESM(require("path"), 1);
259
- var import_node_url3 = require("url");
260
- var grpc2 = __toESM(require("@grpc/grpc-js"), 1);
261
- var import_proto_loader = require("@grpc/proto-loader");
262
- var import_google_proto_files = require("google-proto-files");
263
-
264
- // src/function-executor/safe-output.ts
265
- var nativeJSONStringify = JSON.stringify;
266
- function captureStream(stream) {
267
- stream.on("error", () => void 0);
268
- return {
269
- // Deployed code runs in this process and can replace stream.write. Capture
270
- // the executor's writer before application modules are imported.
271
- write: stream.write.bind(stream)
272
- };
273
- }
274
- var executorOutput = {
275
- stdout: captureStream(process.stdout),
276
- stderr: captureStream(process.stderr)
277
- };
278
- function writeStructuredOutput(destination, createRecord) {
279
- try {
280
- const serialized = nativeJSONStringify(createRecord());
281
- if (serialized == null) return;
282
- executorOutput[destination].write(`${serialized}
283
- `, () => void 0);
284
- } catch {
285
- }
286
- }
287
-
288
- // src/function-executor/service.ts
289
- var import_promises2 = require("fs/promises");
290
- var import_node_os = __toESM(require("os"), 1);
291
- var import_node_path = __toESM(require("path"), 1);
292
- var import_node_url2 = require("url");
293
- var import_node_vm = require("vm");
294
- var grpc = __toESM(require("@grpc/grpc-js"), 1);
295
- var import_fflate = require("fflate");
296
-
297
- // src/applications/errors.ts
298
- var SDK_USAGE_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.sdk-usage-error.v1");
299
- var REQUEST_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.request-error.v1");
300
- var FUNCTION_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.function-error.v1");
301
- var TIMEOUT_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.timeout-error.v1");
302
- var TensorlakeApplicationError = class extends Error {
303
- constructor(message, options) {
304
- super(message, options);
305
- this.name = "TensorlakeApplicationError";
306
- }
307
- };
308
- var SDKUsageError = class _SDKUsageError extends TensorlakeApplicationError {
309
- [SDK_USAGE_ERROR_BRAND] = true;
310
- static [Symbol.hasInstance](value) {
311
- if (this !== _SDKUsageError) {
312
- return Function.prototype[Symbol.hasInstance].call(this, value);
313
- }
314
- return isSDKUsageError(value);
315
- }
316
- constructor(message) {
317
- super(message);
318
- this.name = "SDKUsageError";
319
- }
320
- };
321
- function isSDKUsageError(value) {
322
- if (typeof value !== "object" || value == null) return false;
323
- const candidate = value;
324
- return candidate[SDK_USAGE_ERROR_BRAND] === true && typeof candidate.message === "string";
325
- }
326
- var SerializationError = class extends TensorlakeApplicationError {
327
- constructor(message, options) {
328
- super(message, options);
329
- this.name = "SerializationError";
330
- }
331
- };
332
- var DeserializationError = class extends TensorlakeApplicationError {
333
- constructor(message, options) {
334
- super(message, options);
335
- this.name = "DeserializationError";
336
- }
337
- };
338
- var FunctionError = class _FunctionError extends TensorlakeApplicationError {
339
- [FUNCTION_ERROR_BRAND] = true;
340
- static [Symbol.hasInstance](value) {
341
- if (this !== _FunctionError) {
342
- return Function.prototype[Symbol.hasInstance].call(this, value);
343
- }
344
- return isFunctionError(value);
345
- }
346
- constructor(message, options) {
347
- super(message, options);
348
- this.name = "FunctionError";
349
- }
350
- };
351
- var TimeoutError = class _TimeoutError extends TensorlakeApplicationError {
352
- [TIMEOUT_ERROR_BRAND] = true;
353
- static [Symbol.hasInstance](value) {
354
- if (this !== _TimeoutError) {
355
- return Function.prototype[Symbol.hasInstance].call(this, value);
356
- }
357
- return isTimeoutError(value);
358
- }
359
- constructor() {
360
- super("Timed out.");
361
- this.name = "TimeoutError";
362
- }
363
- };
364
- function isTimeoutError(value) {
365
- if (typeof value !== "object" || value == null) return false;
366
- const candidate = value;
367
- return candidate[TIMEOUT_ERROR_BRAND] === true && typeof candidate.message === "string";
368
- }
369
- function isFunctionError(value) {
370
- if (typeof value !== "object" || value == null) return false;
371
- const candidate = value;
372
- return candidate[FUNCTION_ERROR_BRAND] === true && typeof candidate.message === "string";
373
- }
374
- var RequestError = class _RequestError extends TensorlakeApplicationError {
375
- [REQUEST_ERROR_BRAND] = true;
376
- static [Symbol.hasInstance](value) {
377
- if (this !== _RequestError) {
378
- return Function.prototype[Symbol.hasInstance].call(this, value);
379
- }
380
- return isRequestError(value);
381
- }
382
- constructor(message, options) {
383
- super(message, options);
384
- this.name = "RequestError";
385
- }
386
- };
387
- function isRequestError(value) {
388
- if (typeof value !== "object" || value == null) return false;
389
- const candidate = value;
390
- return candidate[REQUEST_ERROR_BRAND] === true && typeof candidate.message === "string";
391
- }
392
- var ReplayMismatchError = class extends TensorlakeApplicationError {
393
- constructor(message = "Function execution diverged from its durable event history") {
394
- super(message);
395
- this.name = "ReplayMismatchError";
396
- }
397
- };
398
-
399
- // src/applications/registry.ts
400
- var REGISTRY_KEY = /* @__PURE__ */ Symbol.for("tensorlake.applications.registry.v1");
401
- function registry() {
402
- const target = globalThis;
403
- target[REGISTRY_KEY] ??= { functions: /* @__PURE__ */ new Map(), applications: /* @__PURE__ */ new Map() };
404
- return target[REGISTRY_KEY];
405
- }
406
- function snapshotRegistry() {
407
- const state = registry();
408
- return {
409
- functions: new Map(state.functions),
410
- applications: new Map(state.applications)
411
- };
412
- }
413
- function restoreRegistry(snapshot) {
414
- const state = registry();
415
- state.functions.clear();
416
- state.applications.clear();
417
- for (const [name, definition] of snapshot.functions) state.functions.set(name, definition);
418
- for (const [name, definition] of snapshot.applications) state.applications.set(name, definition);
419
- }
420
-
421
- // src/function-executor/allocation.ts
422
- var import_node_async_hooks3 = require("async_hooks");
423
- var import_node_crypto5 = require("crypto");
424
-
425
- // src/applications/function.ts
426
- var import_node_async_hooks = require("async_hooks");
427
- var import_node_crypto = require("crypto");
428
-
429
- // src/applications/schema.ts
430
- var import_ajv = __toESM(require("ajv"), 1);
431
- var import__ = __toESM(require("ajv/dist/2020.js"), 1);
432
-
433
- // src/applications/file.ts
434
- var FILE_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.file.v1");
435
- var File = class _File {
436
- [FILE_BRAND] = true;
437
- content;
438
- contentType;
439
- static [Symbol.hasInstance](value) {
440
- if (this !== _File) {
441
- return Function.prototype[Symbol.hasInstance].call(this, value);
442
- }
443
- return isFile(value);
444
- }
445
- constructor(content, contentType) {
446
- if (!contentType.trim()) {
447
- throw new SDKUsageError("File contentType cannot be empty");
448
- }
449
- this.content = content instanceof Uint8Array ? content : new Uint8Array(content);
450
- this.contentType = contentType;
451
- }
452
- };
453
- function isFile(value) {
454
- if (typeof value !== "object" || value == null) return false;
455
- const candidate = value;
456
- return candidate[FILE_BRAND] === true && candidate.content instanceof Uint8Array && typeof candidate.contentType === "string" && candidate.contentType.trim().length > 0;
457
- }
458
-
459
- // src/applications/http-body.ts
460
- var HTTP_BODY_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.http-body.v1");
461
- var HttpBody = class _HttpBody {
462
- [HTTP_BODY_BRAND] = true;
463
- content;
464
- contentType;
465
- static [Symbol.hasInstance](value) {
466
- if (this !== _HttpBody) {
467
- return Function.prototype[Symbol.hasInstance].call(this, value);
468
- }
469
- return isHttpBody(value);
470
- }
471
- constructor(content, contentType) {
472
- this.content = content instanceof Uint8Array ? content : new Uint8Array(content);
473
- this.contentType = contentType;
474
- }
475
- text(encoding = "utf-8") {
476
- return new TextDecoder(encoding, { fatal: true }).decode(this.content);
477
- }
478
- json() {
479
- return JSON.parse(this.text());
480
- }
481
- };
482
- function isHttpBody(value) {
483
- if (typeof value !== "object" || value == null) return false;
484
- const candidate = value;
485
- return candidate[HTTP_BODY_BRAND] === true && candidate.content instanceof Uint8Array && (candidate.contentType === void 0 || typeof candidate.contentType === "string");
486
- }
487
-
488
- // src/applications/schema.ts
489
- var ajvDraft7 = new import_ajv.default({ allErrors: true, strict: false });
490
- var ajv2020 = new import__.default({ allErrors: true, strict: false });
491
- var validatorCache = /* @__PURE__ */ new WeakMap();
492
- function compileValidator(jsonSchema) {
493
- const dialect = jsonSchema.$schema;
494
- const engine = typeof dialect === "string" && /\/draft-07\/schema#?$/.test(dialect) ? ajvDraft7 : ajv2020;
495
- const validate = engine.compile(jsonSchema);
496
- return {
497
- validate,
498
- errorsText: (errors) => engine.errorsText(errors, { separator: "; " })
499
- };
500
- }
501
- function validateWithSchema(valueSchema, value, label) {
502
- if (valueSchema._file) {
503
- if (!isFile(value)) {
504
- throw new SDKUsageError(`${label} must be a Tensorlake File`);
505
- }
506
- return value;
507
- }
508
- if (valueSchema._httpBody) {
509
- if (!isHttpBody(value)) {
510
- throw new SDKUsageError(`${label} must be a Tensorlake HttpBody`);
511
- }
512
- return value;
513
- }
514
- if (isFile(value)) {
515
- throw new SDKUsageError(`${label} must use schema.file() for a Tensorlake File`);
516
- }
517
- if (isHttpBody(value)) {
518
- throw new SDKUsageError(
519
- `${label} must use schema.httpBody() for a Tensorlake HttpBody`
520
- );
521
- }
522
- let cached = validatorCache.get(valueSchema);
523
- if (cached == null) {
524
- try {
525
- cached = compileValidator(valueSchema.jsonSchema);
526
- } catch (error) {
527
- const details = error instanceof Error ? error.message : String(error);
528
- throw new SDKUsageError(`${label} uses an invalid JSON schema: ${details}`);
529
- }
530
- validatorCache.set(valueSchema, cached);
531
- }
532
- if (!cached.validate(value)) {
533
- const details = cached.errorsText(cached.validate.errors);
534
- throw new SDKUsageError(`${label} does not match its schema: ${details}`);
535
- }
536
- return value;
537
- }
538
-
539
- // src/applications/function.ts
540
- var TAIL_CALL_BRAND = /* @__PURE__ */ Symbol.for(
541
- "tensorlake.applications.tail-call.v1"
542
- );
543
- var DURABLE_PROMISE_OBSERVER = /* @__PURE__ */ Symbol.for(
544
- "tensorlake.applications.durable-promise-observer.v1"
545
- );
546
- var DURABLE_PROMISE_INTERNAL_THEN = /* @__PURE__ */ Symbol.for(
547
- "tensorlake.applications.durable-promise-internal-then.v1"
548
- );
549
- var FUNCTION_RUNTIME_STORAGE_KEY = /* @__PURE__ */ Symbol.for(
550
- "tensorlake.applications.function-runtime-storage.v1"
551
- );
552
- var PROMISE_INSTRUMENTED_KEY = /* @__PURE__ */ Symbol.for(
553
- "tensorlake.applications.promise-instrumented.v3"
554
- );
555
- var PROMISE_THEN_INSTRUMENTED_KEY = /* @__PURE__ */ Symbol.for(
556
- "tensorlake.applications.promise-then-instrumented.v1"
557
- );
558
- var DURABLE_THENABLE_REGISTRY_KEY = /* @__PURE__ */ Symbol.for(
559
- "tensorlake.applications.durable-thenable-registry.v2"
560
- );
561
- var INTERNAL_PROMISE_AGGREGATE_DEPTH_KEY = /* @__PURE__ */ Symbol.for(
562
- "tensorlake.applications.internal-promise-aggregate-depth.v1"
563
- );
564
- function runtimeStorage() {
565
- const target = globalThis;
566
- target[FUNCTION_RUNTIME_STORAGE_KEY] ??= new import_node_async_hooks.AsyncLocalStorage();
567
- return target[FUNCTION_RUNTIME_STORAGE_KEY];
568
- }
569
- function currentFunctionRuntime() {
570
- return runtimeStorage().getStore();
571
- }
572
- function runWithFunctionRuntime(runtime, callback) {
573
- return runtimeStorage().run(runtime, callback);
574
- }
575
- function isDurableThenable(value) {
576
- if (typeof value !== "object" && typeof value !== "function" || value == null) {
577
- return false;
578
- }
579
- return durableThenableRegistry().has(value);
580
- }
581
- function isDetachedDurableThenable(value) {
582
- const provenance = provenanceForDurableThenable(value);
583
- return provenance?.detached === true || provenance?.assimilation != null && !provenance.assimilation.active;
584
- }
585
- function durableThenableRegistry() {
586
- const target = globalThis;
587
- target[DURABLE_THENABLE_REGISTRY_KEY] ??= /* @__PURE__ */ new WeakMap();
588
- return target[DURABLE_THENABLE_REGISTRY_KEY];
589
- }
590
- function newDurableThenableMetadata() {
591
- return {
592
- aggregateExpectations: [],
593
- provenanceExpectations: []
594
- };
595
- }
596
- function metadataForDurableThenable(value) {
597
- return provenanceForDurableThenable(value)?.metadata;
598
- }
599
- function provenanceForDurableThenable(value) {
600
- if (typeof value !== "object" && typeof value !== "function" || value == null) {
601
- return void 0;
602
- }
603
- return durableThenableRegistry().get(value);
604
- }
605
- function registerDurableThenable(value, source) {
606
- const registry2 = durableThenableRegistry();
607
- const metadata = source == null ? registry2.get(value)?.metadata ?? newDurableThenableMetadata() : registry2.get(source)?.metadata ?? newDurableThenableMetadata();
608
- registry2.set(value, { metadata });
609
- if (source != null && !registry2.has(source)) registry2.set(source, { metadata });
610
- return value;
611
- }
612
- function registerDetachedDurableThenable(value) {
613
- const registry2 = durableThenableRegistry();
614
- registry2.set(value, {
615
- metadata: registry2.get(value)?.metadata ?? newDurableThenableMetadata(),
616
- detached: true
617
- });
618
- return value;
619
- }
620
- function inheritDurableThenable(value, source) {
621
- const provenance = durableThenableRegistry().get(source);
622
- if (provenance == null) return value;
623
- durableThenableRegistry().set(value, provenance);
624
- if (provenance.assimilation == null) {
625
- provenance.metadata.provenanceExpectations.push({
626
- value,
627
- creatorAsyncId: (0, import_node_async_hooks.executionAsyncId)()
628
- });
629
- }
630
- return value;
631
- }
632
- function registerDurableAssimilation(value, detach) {
633
- const metadata = metadataForDurableThenable(value);
634
- if (metadata == null) {
635
- return {
636
- finish: () => void 0,
637
- link: (linked) => linked
638
- };
639
- }
640
- const registration = {
641
- active: true,
642
- detach
643
- };
644
- const provenanceIndex = metadata.provenanceExpectations.findIndex(
645
- (expectation) => expectation.creatorAsyncId !== (0, import_node_async_hooks.executionAsyncId)()
646
- );
647
- if (provenanceIndex >= 0) {
648
- const [expectation] = metadata.provenanceExpectations.splice(provenanceIndex, 1);
649
- durableThenableRegistry().set(expectation.value, {
650
- metadata,
651
- assimilation: registration
652
- });
653
- }
654
- const expectationIndex = metadata.aggregateExpectations.findIndex(
655
- (expectation) => !expectation.scope.settled && expectation.creatorAsyncId !== (0, import_node_async_hooks.executionAsyncId)()
656
- );
657
- if (expectationIndex >= 0) {
658
- const [expectation] = metadata.aggregateExpectations.splice(expectationIndex, 1);
659
- associateAssimilationWithAggregate(registration, expectation.scope);
660
- }
661
- return {
662
- finish: () => finishDurableAssimilation(registration),
663
- link: (linked) => {
664
- durableThenableRegistry().set(linked, {
665
- metadata,
666
- assimilation: registration
667
- });
668
- return linked;
669
- }
670
- };
671
- }
672
- function associateAssimilationWithAggregate(registration, scope) {
673
- if (!registration.active || scope.settled) return;
674
- finishDurableAssimilation(registration);
675
- registration.detach();
676
- }
677
- function finishDurableAssimilation(registration) {
678
- if (!registration.active) return;
679
- registration.active = false;
680
- }
681
- function trackDurableAggregate(runtime, result, durableContenders, markResult) {
682
- const scope = {
683
- settled: false,
684
- metadata: /* @__PURE__ */ new Set()
685
- };
686
- if (markResult) {
687
- registerDetachedDurableThenable(result);
688
- }
689
- const creatorAsyncId = (0, import_node_async_hooks.executionAsyncId)();
690
- for (const contender of durableContenders) {
691
- const provenance = provenanceForDurableThenable(contender);
692
- if (provenance == null) continue;
693
- if (provenance.assimilation != null) {
694
- associateAssimilationWithAggregate(provenance.assimilation, scope);
695
- } else {
696
- provenance.metadata.aggregateExpectations.push({
697
- scope,
698
- creatorAsyncId
699
- });
700
- }
701
- if (scope.metadata.has(provenance.metadata)) continue;
702
- const { metadata } = provenance;
703
- scope.metadata.add(metadata);
704
- }
705
- const settle = () => {
706
- if (scope.settled) return;
707
- scope.settled = true;
708
- for (const metadata of scope.metadata) {
709
- for (let index = metadata.aggregateExpectations.length - 1; index >= 0; index -= 1) {
710
- if (metadata.aggregateExpectations[index].scope === scope) {
711
- metadata.aggregateExpectations.splice(index, 1);
712
- }
713
- }
714
- }
715
- scope.metadata.clear();
716
- };
717
- if (runtime.trackPromiseSettlement?.(result, settle) !== true) settle();
718
- }
719
- function promiseAggregateObservation(kind) {
720
- let settleLifetime;
721
- const ordinaryLifetime = new Promise((resolve) => {
722
- settleLifetime = resolve;
723
- });
724
- let contenderCount;
725
- let fulfillmentCount = 0;
726
- let rejectionCount = 0;
727
- let settled = false;
728
- const maybeSettle = () => {
729
- if (settled) return;
730
- const canNoLongerAdvance = kind === "any" ? contenderCount != null && rejectionCount === contenderCount : kind === "all" ? rejectionCount > 0 || contenderCount != null && fulfillmentCount === contenderCount : contenderCount != null && fulfillmentCount + rejectionCount === contenderCount;
731
- if (!canNoLongerAdvance) return;
732
- settled = true;
733
- settleLifetime();
734
- };
735
- return {
736
- ordinaryLifetime,
737
- allOrdinaryValuesObservable: true,
738
- observeFulfillment: () => {
739
- fulfillmentCount += 1;
740
- maybeSettle();
741
- },
742
- observeRejection: () => {
743
- rejectionCount += 1;
744
- maybeSettle();
745
- },
746
- finishIteration: (count) => {
747
- contenderCount = count;
748
- maybeSettle();
749
- }
750
- };
751
- }
752
- function observePromiseAggregateContender(normalized, observation) {
753
- if (typeof normalized !== "object" && typeof normalized !== "function" || normalized == null) {
754
- observation.allOrdinaryValuesObservable = false;
755
- return normalized;
756
- }
757
- let observedSettled = false;
758
- const observed = /* @__PURE__ */ Object.create(null);
759
- Object.defineProperty(observed, "then", {
760
- configurable: true,
761
- get: () => {
762
- const then = Reflect.get(normalized, "then", normalized);
763
- if (typeof then !== "function") return then;
764
- return function observedThen(onfulfilled, onrejected) {
765
- return Reflect.apply(then, normalized, [
766
- function observedFulfillment(value) {
767
- if (!observedSettled) {
768
- observedSettled = true;
769
- observation.observeFulfillment();
770
- }
771
- return typeof onfulfilled === "function" ? Reflect.apply(onfulfilled, this, [value]) : void 0;
772
- },
773
- function observedRejection(reason) {
774
- if (!observedSettled) {
775
- observedSettled = true;
776
- observation.observeRejection();
777
- }
778
- if (typeof onrejected === "function") {
779
- return Reflect.apply(onrejected, this, [reason]);
780
- }
781
- throw reason;
782
- }
783
- ]);
784
- };
785
- }
786
- });
787
- return observed;
788
- }
789
- function observedPromiseConstructor(constructor, observeNormalized) {
790
- const facade = function PromiseAggregateObservedConstructor(executor) {
791
- return Reflect.construct(
792
- constructor,
793
- [executor],
794
- constructor
795
- );
796
- };
797
- Object.defineProperty(facade, "resolve", {
798
- configurable: true,
799
- get: () => {
800
- const promiseResolve = Reflect.get(constructor, "resolve", constructor);
801
- if (typeof promiseResolve !== "function") return promiseResolve;
802
- return (value) => {
803
- const normalized = Reflect.apply(promiseResolve, constructor, [value]);
804
- return observeNormalized(normalized);
805
- };
806
- }
807
- });
808
- return facade;
809
- }
810
- function isInternalPromiseAggregate() {
811
- const target = globalThis;
812
- return (target[INTERNAL_PROMISE_AGGREGATE_DEPTH_KEY] ?? 0) > 0;
813
- }
814
- function installPromiseInstrumentation() {
815
- const target = globalThis;
816
- if (target[PROMISE_THEN_INSTRUMENTED_KEY] !== true) {
817
- const originalThen = Promise.prototype.then;
818
- Object.defineProperty(Promise.prototype, "then", {
819
- ...Object.getOwnPropertyDescriptor(Promise.prototype, "then"),
820
- value: function then(onfulfilled, onrejected) {
821
- const result = Reflect.apply(
822
- originalThen,
823
- this,
824
- [onfulfilled, onrejected]
825
- );
826
- if (isDetachedDurableThenable(this)) {
827
- currentFunctionRuntime()?.trackDurablePromiseConsumption?.(result);
828
- }
829
- return result;
830
- }
831
- });
832
- target[PROMISE_THEN_INSTRUMENTED_KEY] = true;
833
- }
834
- if (target[PROMISE_INSTRUMENTED_KEY] === true) return;
835
- const originalRace = Promise.race;
836
- const originalAny = Promise.any;
837
- const originalAll = Promise.all;
838
- const originalAllSettled = Promise.allSettled;
839
- const originalResolve = Promise.resolve;
840
- Object.defineProperty(Promise, "resolve", {
841
- ...Object.getOwnPropertyDescriptor(Promise, "resolve"),
842
- value: function resolve(value) {
843
- const result = Reflect.apply(originalResolve, this, [value]);
844
- if (isDurableThenable(value)) inheritDurableThenable(result, value);
845
- return result;
846
- }
847
- });
848
- Object.defineProperty(Promise, "race", {
849
- ...Object.getOwnPropertyDescriptor(Promise, "race"),
850
- value: function race(values) {
851
- const runtime = currentFunctionRuntime();
852
- if (runtime?.trackMixedPromiseRace == null) {
853
- return Reflect.apply(originalRace, this, [values]);
854
- }
855
- const durableContenders = [];
856
- let ordinaryContenderCount = 0;
857
- const trackedValues = {
858
- *[Symbol.iterator]() {
859
- for (const value of values) {
860
- yield value;
861
- }
862
- }
863
- };
864
- const observedConstructor = observedPromiseConstructor(
865
- this,
866
- (normalized) => {
867
- if (isDurableThenable(normalized)) {
868
- durableContenders.push(normalized);
869
- } else {
870
- ordinaryContenderCount += 1;
871
- }
872
- return normalized;
873
- }
874
- );
875
- const construct = () => Reflect.apply(
876
- originalRace,
877
- observedConstructor,
878
- [trackedValues]
879
- );
880
- const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
881
- if (durableContenders.length > 0) {
882
- trackDurableAggregate(
883
- runtime,
884
- result,
885
- durableContenders,
886
- !isInternalPromiseAggregate()
887
- );
888
- }
889
- if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
890
- runtime.trackMixedPromiseRace(result);
891
- }
892
- return result;
893
- }
894
- });
895
- Object.defineProperty(Promise, "any", {
896
- ...Object.getOwnPropertyDescriptor(Promise, "any"),
897
- value: function any(values) {
898
- const runtime = currentFunctionRuntime();
899
- if (runtime?.trackMixedPromiseRace == null) {
900
- return Reflect.apply(originalAny, this, [values]);
901
- }
902
- const durableContenders = [];
903
- let ordinaryContenderCount = 0;
904
- const observation = promiseAggregateObservation("any");
905
- const trackedValues = {
906
- *[Symbol.iterator]() {
907
- for (const value of values) {
908
- yield value;
909
- }
910
- }
911
- };
912
- const observedConstructor = observedPromiseConstructor(
913
- this,
914
- (normalized) => {
915
- if (isDurableThenable(normalized)) {
916
- durableContenders.push(normalized);
917
- return normalized;
918
- }
919
- ordinaryContenderCount += 1;
920
- return observePromiseAggregateContender(normalized, observation);
921
- }
922
- );
923
- const construct = () => Reflect.apply(
924
- originalAny,
925
- observedConstructor,
926
- [trackedValues]
927
- );
928
- const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
929
- observation.finishIteration(ordinaryContenderCount);
930
- if (durableContenders.length > 0) {
931
- trackDurableAggregate(
932
- runtime,
933
- result,
934
- durableContenders,
935
- !isInternalPromiseAggregate()
936
- );
937
- }
938
- if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
939
- if (observation.allOrdinaryValuesObservable) {
940
- runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
941
- } else {
942
- runtime.trackMixedPromiseRace(result);
943
- }
944
- }
945
- return result;
946
- }
947
- });
948
- Object.defineProperty(Promise, "all", {
949
- ...Object.getOwnPropertyDescriptor(Promise, "all"),
950
- value: function all(values) {
951
- const runtime = currentFunctionRuntime();
952
- if (runtime?.trackMixedPromiseRace == null) {
953
- return Reflect.apply(originalAll, this, [values]);
954
- }
955
- const durableContenders = [];
956
- let ordinaryContenderCount = 0;
957
- const observation = promiseAggregateObservation("all");
958
- const trackedValues = {
959
- *[Symbol.iterator]() {
960
- for (const value of values) {
961
- yield value;
962
- }
963
- }
964
- };
965
- const observedConstructor = observedPromiseConstructor(
966
- this,
967
- (normalized) => {
968
- if (isDurableThenable(normalized)) {
969
- durableContenders.push(normalized);
970
- return normalized;
971
- }
972
- ordinaryContenderCount += 1;
973
- return observePromiseAggregateContender(normalized, observation);
974
- }
975
- );
976
- const construct = () => Reflect.apply(
977
- originalAll,
978
- observedConstructor,
979
- [trackedValues]
980
- );
981
- const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
982
- observation.finishIteration(ordinaryContenderCount);
983
- if (durableContenders.length > 0) {
984
- trackDurableAggregate(
985
- runtime,
986
- result,
987
- durableContenders,
988
- !isInternalPromiseAggregate()
989
- );
990
- }
991
- if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
992
- if (observation.allOrdinaryValuesObservable) {
993
- runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
994
- } else {
995
- runtime.trackMixedPromiseRace(result);
996
- }
997
- }
998
- return result;
999
- }
1000
- });
1001
- Object.defineProperty(Promise, "allSettled", {
1002
- ...Object.getOwnPropertyDescriptor(Promise, "allSettled"),
1003
- value: function allSettled(values) {
1004
- const runtime = currentFunctionRuntime();
1005
- if (runtime?.trackMixedPromiseRace == null) {
1006
- return Reflect.apply(
1007
- originalAllSettled,
1008
- this,
1009
- [values]
1010
- );
1011
- }
1012
- const durableContenders = [];
1013
- let ordinaryContenderCount = 0;
1014
- const observation = promiseAggregateObservation("all_settled");
1015
- const trackedValues = {
1016
- *[Symbol.iterator]() {
1017
- for (const value of values) yield value;
1018
- }
1019
- };
1020
- const observedConstructor = observedPromiseConstructor(
1021
- this,
1022
- (normalized) => {
1023
- if (isDurableThenable(normalized)) {
1024
- durableContenders.push(normalized);
1025
- return normalized;
1026
- }
1027
- ordinaryContenderCount += 1;
1028
- return observePromiseAggregateContender(normalized, observation);
1029
- }
1030
- );
1031
- const construct = () => Reflect.apply(
1032
- originalAllSettled,
1033
- observedConstructor,
1034
- [trackedValues]
1035
- );
1036
- const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
1037
- observation.finishIteration(ordinaryContenderCount);
1038
- if (durableContenders.length > 0) {
1039
- trackDurableAggregate(
1040
- runtime,
1041
- result,
1042
- durableContenders,
1043
- !isInternalPromiseAggregate()
1044
- );
1045
- }
1046
- if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
1047
- if (observation.allOrdinaryValuesObservable) {
1048
- runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
1049
- } else {
1050
- runtime.trackMixedPromiseRace(result);
1051
- }
1052
- }
1053
- return result;
1054
- }
1055
- });
1056
- target[PROMISE_INSTRUMENTED_KEY] = true;
1057
- }
1058
- function validateArguments(definition, supplied) {
1059
- const args = [...supplied];
1060
- if (args.length > definition.parameters.length) {
1061
- throw new SDKUsageError(`${definition.name} expected at most ${definition.parameters.length} arguments, received ${args.length}`);
1062
- }
1063
- definition.parameters.forEach((parameter, index) => {
1064
- if (index >= args.length || args[index] === void 0) {
1065
- if (parameter.hasDefault) args[index] = parameter.defaultValue;
1066
- else if (parameter.required) throw new SDKUsageError(`${definition.name} is missing argument '${parameter.name}'`);
1067
- return;
1068
- }
1069
- validateWithSchema(parameter.schema, args[index], `${definition.name}.${parameter.name}`);
1070
- });
1071
- return args;
1072
- }
1073
- async function executeHandlerResult(definition, suppliedArgs) {
1074
- const args = validateArguments(definition, suppliedArgs);
1075
- const result = definition.handler(...args);
1076
- if (result == null || typeof result.then !== "function") {
1077
- throw new SDKUsageError(`Tensorlake handler '${definition.name}' must be async and return a Promise`);
1078
- }
1079
- const output = await result;
1080
- if (isTailCall(output)) {
1081
- return output;
1082
- }
1083
- validateWithSchema(definition.returns, output, `${definition.name} return value`);
1084
- return output;
1085
- }
1086
- function isTailCall(value) {
1087
- if (typeof value !== "object" || value == null) return false;
1088
- const candidate = value;
1089
- return candidate[TAIL_CALL_BRAND] === true && candidate.kind === "tensorlake-tail-call" && typeof candidate.definition === "object" && candidate.definition != null && Array.isArray(candidate.args);
1090
- }
1091
-
1092
- // src/applications/headers.ts
1093
- var HEADERS_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.headers.v1");
1094
- var Headers2 = class _Headers {
1095
- [HEADERS_BRAND] = true;
1096
- #items;
1097
- #lookup;
1098
- static [Symbol.hasInstance](value) {
1099
- if (this !== _Headers) {
1100
- return Function.prototype[Symbol.hasInstance].call(this, value);
1101
- }
1102
- return isHeaders(value);
1103
- }
1104
- constructor(headers = []) {
1105
- const entries = headers instanceof _Headers ? [...headers] : Symbol.iterator in Object(headers) ? [...headers] : Object.entries(headers);
1106
- const lookup = /* @__PURE__ */ new Map();
1107
- this.#items = Object.freeze(entries.map(([name, value]) => {
1108
- if (typeof name !== "string" || typeof value !== "string") {
1109
- throw new TypeError("Headers require string names and values");
1110
- }
1111
- const item = Object.freeze([name, value]);
1112
- const key = name.toLowerCase();
1113
- lookup.set(key, [...lookup.get(key) ?? [], value]);
1114
- return item;
1115
- }));
1116
- this.#lookup = lookup;
1117
- Object.freeze(this);
1118
- }
1119
- get(name) {
1120
- return this.#lookup.get(name.toLowerCase())?.at(-1);
1121
- }
1122
- getAll(name) {
1123
- return Object.freeze([...this.#lookup.get(name.toLowerCase()) ?? []]);
1124
- }
1125
- has(name) {
1126
- return this.#lookup.has(name.toLowerCase());
1127
- }
1128
- entries() {
1129
- return this.#items[Symbol.iterator]();
1130
- }
1131
- [Symbol.iterator]() {
1132
- return this.entries();
1133
- }
1134
- };
1135
- function isHeaders(value) {
1136
- if (typeof value !== "object" || value == null) return false;
1137
- const candidate = value;
1138
- return candidate[HEADERS_BRAND] === true && typeof candidate.get === "function" && typeof candidate.getAll === "function" && typeof candidate.has === "function" && typeof candidate[Symbol.iterator] === "function";
1139
- }
1140
-
1141
- // src/applications/context.ts
1142
- var import_node_async_hooks2 = require("async_hooks");
1143
-
1144
- // src/applications/serialization.ts
1145
- var import_node_crypto2 = require("crypto");
1146
- var textEncoder = new TextEncoder();
1147
- var textDecoder = new TextDecoder("utf-8", { fatal: true });
1148
- function assertJSONValue(value, path3, seen) {
1149
- if (value === null || typeof value === "string" || typeof value === "boolean") return;
1150
- if (typeof value === "number") {
1151
- if (!Number.isFinite(value)) throw new SerializationError(`${path3} contains a non-finite number`);
1152
- return;
1153
- }
1154
- if (typeof value !== "object") {
1155
- throw new SerializationError(`${path3} contains unsupported ${typeof value}`);
1156
- }
1157
- if (isFile(value)) {
1158
- throw new SerializationError(`${path3} contains a nested File; File is only supported as a direct value`);
1159
- }
1160
- if (isHttpBody(value)) {
1161
- throw new SerializationError(
1162
- `${path3} contains a nested HttpBody; HttpBody is only supported as a direct value`
1163
- );
1164
- }
1165
- if (seen.has(value)) throw new SerializationError(`${path3} contains a cycle`);
1166
- if (Object.getPrototypeOf(value) !== Object.prototype && !Array.isArray(value)) {
1167
- throw new SerializationError(`${path3} contains unsupported ${value.constructor?.name ?? "object"}`);
1168
- }
1169
- seen.add(value);
1170
- if (Array.isArray(value)) {
1171
- for (let index = 0; index < value.length; index += 1) {
1172
- if (!Object.prototype.hasOwnProperty.call(value, index)) {
1173
- throw new SerializationError(`${path3} contains a sparse array entry at index ${index}`);
1174
- }
1175
- assertJSONValue(value[index], `${path3}[${index}]`, seen);
1176
- }
1177
- } else {
1178
- for (const [key, item] of Object.entries(value)) {
1179
- assertJSONValue(item, `${path3}.${key}`, seen);
1180
- }
1181
- }
1182
- seen.delete(value);
1183
- }
1184
- function serializeValue(value) {
1185
- if (isFile(value)) {
1186
- return { data: value.content, contentType: value.contentType, encoding: "raw" };
1187
- }
1188
- if (isHttpBody(value)) {
1189
- return {
1190
- data: value.content,
1191
- contentType: value.contentType ?? "",
1192
- encoding: "raw"
1193
- };
1194
- }
1195
- try {
1196
- assertJSONValue(value, "value", /* @__PURE__ */ new Set());
1197
- return {
1198
- data: textEncoder.encode(JSON.stringify(value)),
1199
- contentType: "application/json; charset=UTF-8",
1200
- encoding: "json"
1201
- };
1202
- } catch (error) {
1203
- if (error instanceof SerializationError) throw error;
1204
- throw new SerializationError("Failed to serialize JSON value", { cause: error });
1205
- }
1206
- }
1207
- function deserializeJSON(data) {
1208
- try {
1209
- return JSON.parse(textDecoder.decode(data));
1210
- } catch (error) {
1211
- throw new DeserializationError("Failed to deserialize JSON value", { cause: error });
1212
- }
1213
- }
1214
-
1215
- // src/applications/context.ts
1216
- function validateRequestStateKey(key) {
1217
- if (typeof key !== "string") {
1218
- throw new SDKUsageError(`State key must be a string, got: ${String(key)}`);
1219
- }
1220
- }
1221
- function serializeRequestStateValue(value) {
1222
- const serialized = serializeValue(value);
1223
- if (serialized.encoding !== "json") {
1224
- throw new SDKUsageError("Request state values must be JSON values");
1225
- }
1226
- return serialized.data;
1227
- }
1228
- function validateCounterMetric(name, value) {
1229
- if (typeof name !== "string") {
1230
- throw new SDKUsageError(`Counter name must be a string, got: ${String(name)}`);
1231
- }
1232
- if (typeof value !== "number" || !Number.isInteger(value)) {
1233
- throw new SDKUsageError(`Counter value must be an int, got: ${String(value)}`);
1234
- }
1235
- if (!Number.isSafeInteger(value)) {
1236
- throw new SDKUsageError(`Counter value must be a safe int, got: ${String(value)}`);
1237
- }
1238
- }
1239
- function validateTimerMetric(name, value) {
1240
- if (typeof name !== "string") {
1241
- throw new SDKUsageError(`Timer name must be a string, got: ${String(name)}`);
1242
- }
1243
- if (typeof value !== "number" || !Number.isFinite(value)) {
1244
- throw new SDKUsageError(`Timer value must be a finite number, got: ${String(value)}`);
1245
- }
1246
- }
1247
- function isPlainRecord(value) {
1248
- if (value == null || typeof value !== "object" || Array.isArray(value)) return false;
1249
- const prototype = Object.getPrototypeOf(value);
1250
- return prototype === Object.prototype || prototype === null;
1251
- }
1252
- function validateProgressUpdate(current, total, options) {
1253
- if (!Number.isFinite(current) || !Number.isFinite(total) || total < 0 || current < 0) {
1254
- throw new SDKUsageError("Progress current and total must be non-negative finite numbers");
1255
- }
1256
- if (options != null && !isPlainRecord(options)) {
1257
- throw new SDKUsageError("Progress options must be an object");
1258
- }
1259
- if (options?.message != null && typeof options.message !== "string") {
1260
- throw new SDKUsageError("Progress message must be a string");
1261
- }
1262
- if (options?.attributes != null) {
1263
- if (!isPlainRecord(options.attributes)) {
1264
- throw new SDKUsageError("Progress attributes must be an object of string key/value pairs");
1265
- }
1266
- for (const key of Reflect.ownKeys(options.attributes)) {
1267
- if (typeof key !== "string") {
1268
- throw new SDKUsageError("Progress attributes must contain only string keys and values");
1269
- }
1270
- const value = options.attributes[key];
1271
- if (typeof value !== "string") {
1272
- throw new SDKUsageError("Progress attributes must contain only string keys and values");
1273
- }
1274
- }
1275
- }
1276
- }
1277
- var REQUEST_CONTEXT_STORAGE_KEY = /* @__PURE__ */ Symbol.for(
1278
- "tensorlake.applications.request-context-storage.v1"
1279
- );
1280
- function requestContextStorage() {
1281
- const target = globalThis;
1282
- target[REQUEST_CONTEXT_STORAGE_KEY] ??= new import_node_async_hooks2.AsyncLocalStorage();
1283
- return target[REQUEST_CONTEXT_STORAGE_KEY];
1284
- }
1285
- function runWithRequestContext(context, callback) {
1286
- return requestContextStorage().run(context, callback);
1287
- }
1288
- async function waitWithAbortSignal(value, signal) {
1289
- if (signal.aborted) throw signal.reason;
1290
- let onAbort;
1291
- try {
1292
- return await Promise.race([
1293
- value,
1294
- new Promise((_resolve, reject) => {
1295
- onAbort = () => reject(signal.reason);
1296
- signal.addEventListener("abort", onAbort, { once: true });
1297
- })
1298
- ]);
1299
- } finally {
1300
- if (onAbort != null) signal.removeEventListener("abort", onAbort);
1301
- }
1302
- }
1303
-
1304
- // src/function-executor/async-queue.ts
1305
- function deferred() {
1306
- let resolve;
1307
- let reject;
1308
- const promise = new Promise((ok, fail) => {
1309
- resolve = ok;
1310
- reject = fail;
1311
- });
1312
- return { promise, resolve, reject };
1313
- }
1314
-
1315
- // src/function-executor/blob.ts
1316
- var import_node_crypto3 = require("crypto");
1317
- var import_promises = require("fs/promises");
1318
- var import_node_url = require("url");
1319
- function enumIs(value, name, numberValue) {
1320
- return value === name || value === numberValue;
1321
- }
1322
- function concatenate(parts) {
1323
- const result = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
1324
- let offset = 0;
1325
- for (const part of parts) {
1326
- result.set(part, offset);
1327
- offset += part.byteLength;
1328
- }
1329
- return result;
1330
- }
1331
- function networkURI(uri) {
1332
- if (uri.startsWith("s3://")) return `https://${uri.slice("s3://".length)}`;
1333
- if (uri.startsWith("gs://")) return `https://${uri.slice("gs://".length)}`;
1334
- return uri;
1335
- }
1336
- var BlobHTTPError = class extends Error {
1337
- constructor(status2) {
1338
- super(`BLOB operation failed with HTTP ${status2}`);
1339
- this.status = status2;
1340
- }
1341
- };
1342
- function uriKind(uri) {
1343
- if (uri.startsWith("file://")) return "file";
1344
- if (uri.startsWith("s3://")) return "s3";
1345
- if (uri.startsWith("gs://")) return "gcs";
1346
- try {
1347
- return new URL(uri).protocol.replace(":", "") || "unknown";
1348
- } catch {
1349
- return "unknown";
1350
- }
1351
- }
1352
- function throwIfAborted(signal) {
1353
- signal?.throwIfAborted();
1354
- }
1355
- async function abortableDelay(delayMs, signal) {
1356
- throwIfAborted(signal);
1357
- if (signal == null) {
1358
- await new Promise((resolve) => setTimeout(resolve, delayMs));
1359
- return;
1360
- }
1361
- await new Promise((resolve, reject) => {
1362
- const timer = setTimeout(() => {
1363
- signal.removeEventListener("abort", onAbort);
1364
- resolve();
1365
- }, delayMs);
1366
- const onAbort = () => {
1367
- clearTimeout(timer);
1368
- reject(signal.reason);
1369
- };
1370
- signal.addEventListener("abort", onAbort, { once: true });
1371
- });
1372
- }
1373
- async function fetchWithRetries(uri, init, log, fields = {}, signal) {
1374
- let lastError;
1375
- for (let attempt = 0; attempt < 4; attempt += 1) {
1376
- throwIfAborted(signal);
1377
- const startedAt = Date.now();
1378
- log?.("HTTP attempt starting", {
1379
- ...fields,
1380
- attempt: attempt + 1,
1381
- method: init?.method ?? "GET",
1382
- uri_kind: uriKind(uri),
1383
- has_range: new Headers(init?.headers).has("range")
1384
- });
1385
- try {
1386
- const response = await fetch(networkURI(uri), {
1387
- ...init,
1388
- signal
1389
- });
1390
- log?.("HTTP attempt completed", {
1391
- ...fields,
1392
- attempt: attempt + 1,
1393
- method: init?.method ?? "GET",
1394
- status_code: response.status,
1395
- duration_ms: Date.now() - startedAt
1396
- });
1397
- throwIfAborted(signal);
1398
- if (response.ok) return response;
1399
- if ([400, 403, 404].includes(response.status) || attempt === 3) {
1400
- throw new BlobHTTPError(response.status);
1401
- }
1402
- lastError = new BlobHTTPError(response.status);
1403
- } catch (error) {
1404
- log?.("HTTP attempt failed", {
1405
- ...fields,
1406
- attempt: attempt + 1,
1407
- method: init?.method ?? "GET",
1408
- duration_ms: Date.now() - startedAt,
1409
- error: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
1410
- });
1411
- throwIfAborted(signal);
1412
- if (error instanceof BlobHTTPError && [400, 403, 404].includes(error.status)) throw error;
1413
- lastError = error;
1414
- if (attempt === 3) break;
1415
- }
1416
- const retryDelayMs = 100 * 2 ** attempt;
1417
- log?.("HTTP attempt scheduled for retry", {
1418
- ...fields,
1419
- next_attempt: attempt + 2,
1420
- retry_delay_ms: retryDelayMs
1421
- });
1422
- await abortableDelay(retryDelayMs, signal);
1423
- }
1424
- throw new Error("BLOB operation failed after retries", { cause: lastError });
1425
- }
1426
- async function readChunkRange(chunk, index, logicalChunkOffset, offsetWithinChunk, size, sharedURI, log, signal) {
1427
- throwIfAborted(signal);
1428
- if (!chunk.uri) throw new Error("BLOB chunk has no URI");
1429
- const physicalOffset = sharedURI ? logicalChunkOffset + offsetWithinChunk : offsetWithinChunk;
1430
- const fields = {
1431
- chunk_index: index,
1432
- chunk_offset: logicalChunkOffset,
1433
- read_offset: physicalOffset,
1434
- read_bytes: size,
1435
- uri_kind: uriKind(chunk.uri),
1436
- shared_uri: sharedURI
1437
- };
1438
- const startedAt = Date.now();
1439
- log?.("download chunk starting", fields);
1440
- if (chunk.uri.startsWith("file://")) {
1441
- const file = await (0, import_promises.open)((0, import_node_url.fileURLToPath)(chunk.uri), "r");
1442
- try {
1443
- throwIfAborted(signal);
1444
- const data2 = new Uint8Array(size);
1445
- const { bytesRead } = await file.read(data2, 0, size, physicalOffset);
1446
- throwIfAborted(signal);
1447
- if (bytesRead !== size) {
1448
- throw new Error(`BLOB chunk ended after ${bytesRead} bytes; expected ${size}`);
1449
- }
1450
- log?.("download chunk completed", {
1451
- ...fields,
1452
- downloaded_bytes: data2.byteLength,
1453
- duration_ms: Date.now() - startedAt
1454
- });
1455
- return data2;
1456
- } finally {
1457
- await file.close();
1458
- }
1459
- }
1460
- const headers = { range: `bytes=${physicalOffset}-${physicalOffset + size - 1}` };
1461
- const response = await fetchWithRetries(chunk.uri, { headers }, log, fields, signal);
1462
- let data = new Uint8Array(await response.arrayBuffer());
1463
- throwIfAborted(signal);
1464
- if (response.status === 200 && (physicalOffset !== 0 || data.byteLength !== size)) {
1465
- data = data.subarray(physicalOffset, physicalOffset + size);
1466
- }
1467
- if (data.byteLength !== size) {
1468
- throw new Error(`BLOB chunk returned ${data.byteLength} bytes; expected ${size}`);
1469
- }
1470
- log?.("download chunk completed", {
1471
- ...fields,
1472
- status_code: response.status,
1473
- downloaded_bytes: data.byteLength,
1474
- duration_ms: Date.now() - startedAt
1475
- });
1476
- return data;
1477
- }
1478
- async function downloadBlobRange(blob, offset, size, log, signal) {
1479
- throwIfAborted(signal);
1480
- if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(size) || size < 0) {
1481
- throw new Error("BLOB range offset and size must be non-negative safe integers");
1482
- }
1483
- const chunks = blob.chunks ?? [];
1484
- const totalSize = chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0);
1485
- if (offset + size > totalSize) {
1486
- throw new Error(`BLOB range ${offset}..${offset + size} exceeds declared size ${totalSize}`);
1487
- }
1488
- if (size === 0) return new Uint8Array();
1489
- const uriCounts = /* @__PURE__ */ new Map();
1490
- for (const chunk of chunks) {
1491
- if (chunk.uri != null) uriCounts.set(chunk.uri, (uriCounts.get(chunk.uri) ?? 0) + 1);
1492
- }
1493
- const rangeEnd = offset + size;
1494
- let chunkOffset = 0;
1495
- const reads = [];
1496
- for (const [index, chunk] of chunks.entries()) {
1497
- const chunkSize = chunk.size ?? 0;
1498
- const chunkEnd = chunkOffset + chunkSize;
1499
- const overlapStart = Math.max(offset, chunkOffset);
1500
- const overlapEnd = Math.min(rangeEnd, chunkEnd);
1501
- if (overlapStart < overlapEnd) {
1502
- reads.push(readChunkRange(
1503
- chunk,
1504
- index,
1505
- chunkOffset,
1506
- overlapStart - chunkOffset,
1507
- overlapEnd - overlapStart,
1508
- (uriCounts.get(chunk.uri ?? "") ?? 0) > 1,
1509
- log,
1510
- signal
1511
- ));
1512
- }
1513
- chunkOffset = chunkEnd;
1514
- }
1515
- const data = concatenate(await Promise.all(reads));
1516
- throwIfAborted(signal);
1517
- if (data.byteLength !== size) throw new Error("BLOB does not contain the complete requested range");
1518
- log?.("download blob completed", {
1519
- blob_id: blob.id,
1520
- chunk_count: chunks.length,
1521
- downloaded_bytes: data.byteLength,
1522
- range_offset: offset,
1523
- range_size: size
1524
- });
1525
- return data;
1526
- }
1527
- async function downloadBlob(blob, log, signal) {
1528
- const size = (blob.chunks ?? []).reduce((total, chunk) => total + (chunk.size ?? 0), 0);
1529
- return downloadBlobRange(blob, 0, size, log, signal);
1530
- }
1531
- async function downloadSerializedObject(object, blob, log, signal) {
1532
- throwIfAborted(signal);
1533
- const manifest = object.manifest;
1534
- if (manifest?.size == null || manifest.metadataSize == null) {
1535
- throw new Error("Serialized object manifest is missing size or metadata_size");
1536
- }
1537
- if (!Number.isSafeInteger(manifest.size) || manifest.size < 0) {
1538
- throw new Error("Serialized object size must be a non-negative safe integer");
1539
- }
1540
- if (!Number.isSafeInteger(manifest.metadataSize) || manifest.metadataSize < 0) {
1541
- throw new Error("Serialized object metadata_size must be a non-negative safe integer");
1542
- }
1543
- if (manifest.metadataSize > manifest.size) {
1544
- throw new Error("Serialized object metadata_size cannot exceed size");
1545
- }
1546
- if (typeof manifest.sha256Hash !== "string" || manifest.sha256Hash.length === 0) {
1547
- throw new Error("Serialized object manifest is missing sha256_hash");
1548
- }
1549
- const start = object.offset ?? 0;
1550
- const bytes = await downloadBlobRange(blob, start, manifest.size, log, signal);
1551
- throwIfAborted(signal);
1552
- const hash = (0, import_node_crypto3.createHash)("sha256").update(bytes).digest("hex");
1553
- if (hash !== manifest.sha256Hash) {
1554
- throw new Error(`Serialized object hash mismatch: expected ${manifest.sha256Hash}, got ${hash}`);
1555
- }
1556
- return {
1557
- metadata: bytes.subarray(0, manifest.metadataSize),
1558
- data: bytes.subarray(manifest.metadataSize),
1559
- contentType: manifest.contentType,
1560
- encoding: manifest.encoding
1561
- };
1562
- }
1563
- function deserializeValueFromProtocol(value) {
1564
- if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_RAW", 5)) {
1565
- return new File(value.data, value.contentType ?? "application/octet-stream");
1566
- }
1567
- if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_UTF8_TEXT", 2)) {
1568
- return new TextDecoder("utf-8", { fatal: true }).decode(value.data);
1569
- }
1570
- if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_UTF8_JSON", 1) || value.contentType?.toLowerCase().includes("json")) {
1571
- return deserializeJSON(value.data);
1572
- }
1573
- throw new Error(`Unsupported serialized object encoding '${String(value.encoding)}'`);
1574
- }
1575
- function prepareSerializedObject(value, offset = 0, sourceFunctionCallId) {
1576
- const serialized = serializeValue(value);
1577
- const metadata = new TextEncoder().encode(JSON.stringify({
1578
- format: "tensorlake.typescript.value.v1"
1579
- }));
1580
- const bytes = concatenate([metadata, serialized.data]);
1581
- return {
1582
- bytes,
1583
- object: {
1584
- offset,
1585
- manifest: {
1586
- encoding: serialized.encoding === "json" ? "SERIALIZED_OBJECT_ENCODING_UTF8_JSON" : "SERIALIZED_OBJECT_ENCODING_RAW",
1587
- encodingVersion: 0,
1588
- size: bytes.byteLength,
1589
- metadataSize: metadata.byteLength,
1590
- sha256Hash: (0, import_node_crypto3.createHash)("sha256").update(bytes).digest("hex"),
1591
- ...serialized.encoding === "raw" ? { contentType: serialized.contentType } : {},
1592
- sourceFunctionCallId
1593
- }
1594
- }
1595
- };
1596
- }
1597
- function prepareTextObject(value) {
1598
- const bytes = new TextEncoder().encode(value);
1599
- return {
1600
- bytes,
1601
- object: {
1602
- offset: 0,
1603
- manifest: {
1604
- encoding: "SERIALIZED_OBJECT_ENCODING_UTF8_TEXT",
1605
- encodingVersion: 0,
1606
- size: bytes.byteLength,
1607
- metadataSize: 0,
1608
- sha256Hash: (0, import_node_crypto3.createHash)("sha256").update(bytes).digest("hex")
1609
- }
1610
- }
1611
- };
1612
- }
1613
- async function uploadChunk(chunk, data, index, offset, log, signal) {
1614
- throwIfAborted(signal);
1615
- if (!chunk.uri) throw new Error("Writable BLOB chunk has no URI");
1616
- const fields = {
1617
- chunk_index: index,
1618
- chunk_offset: offset,
1619
- upload_bytes: data.byteLength,
1620
- chunk_capacity: chunk.size ?? 0,
1621
- uri_kind: uriKind(chunk.uri)
1622
- };
1623
- const startedAt = Date.now();
1624
- log?.("upload chunk starting", fields);
1625
- if (chunk.uri.startsWith("file://")) {
1626
- const filePath = (0, import_node_url.fileURLToPath)(chunk.uri);
1627
- let file;
1628
- try {
1629
- file = await (0, import_promises.open)(filePath, "r+");
1630
- } catch (error) {
1631
- if (error.code !== "ENOENT") throw error;
1632
- file = await (0, import_promises.open)(filePath, "w+");
1633
- }
1634
- try {
1635
- throwIfAborted(signal);
1636
- await file.write(data, 0, data.byteLength, offset);
1637
- throwIfAborted(signal);
1638
- } finally {
1639
- await file.close();
1640
- }
1641
- log?.("upload chunk completed", { ...fields, duration_ms: Date.now() - startedAt });
1642
- return { ...chunk, size: data.byteLength };
1643
- }
1644
- const response = await fetchWithRetries(chunk.uri, {
1645
- method: "PUT",
1646
- ...data.byteLength === 0 ? {} : {
1647
- body: Uint8Array.from(data).buffer
1648
- }
1649
- }, log, fields, signal);
1650
- throwIfAborted(signal);
1651
- const uploaded = {
1652
- ...chunk,
1653
- size: data.byteLength,
1654
- etag: response.headers.get("etag") ?? void 0
1655
- };
1656
- log?.("upload chunk completed", {
1657
- ...fields,
1658
- status_code: response.status,
1659
- has_etag: uploaded.etag != null,
1660
- duration_ms: Date.now() - startedAt
1661
- });
1662
- return uploaded;
1663
- }
1664
- async function uploadBlob(blob, data, log, signal) {
1665
- throwIfAborted(signal);
1666
- const chunks = blob.chunks ?? [];
1667
- const capacity = chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0);
1668
- if (data.byteLength > capacity) throw new Error(`BLOB capacity ${capacity} is smaller than ${data.byteLength}`);
1669
- let offset = 0;
1670
- const uploaded = [];
1671
- for (const [index, chunk] of chunks.entries()) {
1672
- const size = Math.min(chunk.size ?? 0, data.byteLength - offset);
1673
- if (size <= 0) break;
1674
- uploaded.push(await uploadChunk(
1675
- chunk,
1676
- data.subarray(offset, offset + size),
1677
- index,
1678
- offset,
1679
- log,
1680
- signal
1681
- ));
1682
- offset += size;
1683
- }
1684
- if (data.byteLength === 0 && chunks.length > 0) {
1685
- uploaded.push(await uploadChunk(chunks[0], data, 0, 0, log, signal));
1686
- }
1687
- throwIfAborted(signal);
1688
- if (offset !== data.byteLength) throw new Error(`Only uploaded ${offset} of ${data.byteLength} bytes`);
1689
- log?.("upload blob completed", {
1690
- blob_id: blob.id,
1691
- chunk_count: uploaded.length,
1692
- uploaded_bytes: data.byteLength
1693
- });
1694
- return { ...blob, chunks: uploaded };
1695
- }
1696
- function joinPrepared(values) {
1697
- return concatenate(values.map((value) => value.bytes));
1698
- }
1699
-
1700
- // src/function-executor/user-events.ts
1701
- var import_node_crypto4 = require("crypto");
1702
- function printCloudEvent(data, options = {}) {
1703
- writeStructuredOutput("stdout", () => ({
1704
- specversion: "1.0",
1705
- id: (0, import_node_crypto4.randomUUID)(),
1706
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
1707
- type: options.type ?? "ai.tensorlake.event",
1708
- source: options.source ?? "/tensorlake/function_executor/events",
1709
- data,
1710
- ...options.message == null ? {} : { message: options.message }
1711
- }));
1712
- }
1713
-
1714
- // src/function-executor/allocation.ts
1715
- installPromiseInstrumentation();
1716
- var OK = 0;
1717
- var NOT_FOUND = 5;
1718
- var REPLAY_STRICT = /* @__PURE__ */ new Set(["REPLAY_MODE_STRICT", 1]);
1719
- var OUTCOME_SUCCESS = /* @__PURE__ */ new Set(["ALLOCATION_OUTCOME_CODE_SUCCESS", 1]);
1720
- var OUTCOME_FAILURE = /* @__PURE__ */ new Set(["ALLOCATION_OUTCOME_CODE_FAILURE", 2]);
1721
- var WATCHER_TIMED_OUT = /* @__PURE__ */ new Set(["FUNCTION_CALL_WATCHER_STATUS_TIMEDOUT", 2]);
1722
- var MAX_REDUCE_CALLS_PER_PLAN = 512;
1723
- var nativeStructuredClone = globalThis.structuredClone;
1724
- function statusError(status2, fallback) {
1725
- if (status2 == null || (status2.code ?? OK) === OK) return void 0;
1726
- return new FunctionError(status2.message || fallback);
1727
- }
1728
- function validatedEventLogClock(response, requestedAfterClock, createError) {
1729
- if (response.entries != null && !Array.isArray(response.entries)) {
1730
- throw createError("Allocation event log returned a non-array entries field");
1731
- }
1732
- const responseClock = Number(response.lastClock ?? requestedAfterClock);
1733
- if (!Number.isFinite(responseClock) || !Number.isSafeInteger(responseClock) || responseClock < 0) {
1734
- throw createError(
1735
- `Allocation event log returned invalid clock '${String(response.lastClock)}'`
1736
- );
1737
- }
1738
- if (responseClock < requestedAfterClock) {
1739
- throw createError(
1740
- `Allocation event log moved backwards from clock ${requestedAfterClock} to ${responseClock}`
1741
- );
1742
- }
1743
- if (response.hasMore && responseClock === requestedAfterClock) {
1744
- throw createError(
1745
- `Allocation event log reported more entries without advancing clock ${requestedAfterClock}`
1746
- );
1747
- }
1748
- if ((response.entries?.length ?? 0) > 0 && responseClock === requestedAfterClock) {
1749
- throw createError(
1750
- `Allocation event log returned entries without advancing clock ${requestedAfterClock}`
1751
- );
1752
- }
1753
- const eventFields = [
1754
- "functionCallCreated",
1755
- "functionCallWatcherCreated",
1756
- "functionCallWatcherResult"
1757
- ];
1758
- let previousEntryClock = requestedAfterClock;
1759
- for (const [index, entry] of (response.entries ?? []).entries()) {
1760
- if (entry == null || typeof entry !== "object") {
1761
- throw createError(`Allocation event log entry ${index} is not an object`);
1762
- }
1763
- const presentFields = eventFields.filter((field) => entry[field] != null);
1764
- if (presentFields.length !== 1) {
1765
- throw createError(
1766
- `Allocation event log entry ${index} has ${presentFields.length} recognized payloads; expected exactly one`
1767
- );
1768
- }
1769
- const entryClock = Number(entry.clock);
1770
- if (!Number.isSafeInteger(entryClock) || entryClock <= previousEntryClock || entryClock > responseClock) {
1771
- throw createError(
1772
- `Allocation event log entry ${index} has invalid clock '${String(entry.clock)}' after ${previousEntryClock} with page clock ${responseClock}`
1773
- );
1774
- }
1775
- previousEntryClock = entryClock;
1776
- }
1777
- return responseClock;
1778
- }
1779
- function durableHash(values) {
1780
- const hash = (0, import_node_crypto5.createHash)("sha256");
1781
- for (const value of values) {
1782
- const bytes = Buffer.from(value, "utf8");
1783
- const length = Buffer.alloc(8);
1784
- length.writeBigUInt64BE(BigInt(bytes.byteLength));
1785
- hash.update(length);
1786
- hash.update(bytes);
1787
- hash.update("|");
1788
- }
1789
- return hash.digest("hex");
1790
- }
1791
- function protocolTimestamp(date) {
1792
- const milliseconds = date.getTime();
1793
- return { seconds: Math.floor(milliseconds / 1e3), nanos: milliseconds % 1e3 * 1e6 };
1794
- }
1795
- function parseHTTPMessage(data) {
1796
- const marker = Buffer.from("\r\n\r\n");
1797
- const bytes = Buffer.from(data);
1798
- const split = bytes.indexOf(marker);
1799
- if (split < 0) throw new DeserializationError("Invalid message/http application input");
1800
- const lines = bytes.subarray(0, split).toString("utf8").split("\r\n");
1801
- lines.shift();
1802
- const headers = {};
1803
- for (const line of lines) {
1804
- const colon = line.indexOf(":");
1805
- if (colon > 0) headers[line.slice(0, colon).trim().toLowerCase()] = line.slice(colon + 1).trim();
1806
- }
1807
- return { headers, body: new Uint8Array(bytes.subarray(split + marker.byteLength)) };
1808
- }
1809
- async function deserializeApplicationArguments(definition, payload) {
1810
- let data = payload.data;
1811
- let suppliedContentType = payload.contentType;
1812
- let hasExplicitContentType = payload.contentType != null;
1813
- if (suppliedContentType?.toLowerCase().startsWith("message/http")) {
1814
- const parsed = parseHTTPMessage(data);
1815
- data = parsed.body;
1816
- const bodyContentType = parsed.headers["content-type"];
1817
- hasExplicitContentType = bodyContentType != null;
1818
- suppliedContentType = bodyContentType;
1819
- }
1820
- if (definition.parameters.length === 0) return { args: [] };
1821
- const firstParameter = definition.parameters[0];
1822
- if (definition.parameters.length === 1 && firstParameter?.schema._httpBody) {
1823
- return { args: [new HttpBody(data, suppliedContentType)] };
1824
- }
1825
- const contentType = suppliedContentType ?? "application/json";
1826
- if (contentType.toLowerCase().startsWith("multipart/form-data")) {
1827
- let form;
1828
- try {
1829
- form = await new Response(Uint8Array.from(data).buffer, {
1830
- headers: { "content-type": contentType }
1831
- }).formData();
1832
- } catch (error) {
1833
- throw new DeserializationError("Failed to deserialize multipart application input", { cause: error });
1834
- }
1835
- const args = [];
1836
- for (const [index, parameter] of definition.parameters.entries()) {
1837
- const part = form.get(parameter.name) ?? form.get(String(index));
1838
- if (part == null) {
1839
- args.push(void 0);
1840
- } else if (parameter.schema._file || parameter.schema._httpBody) {
1841
- const bytes = typeof part === "string" ? new TextEncoder().encode(part) : new Uint8Array(await part.arrayBuffer());
1842
- const partContentType = typeof part === "string" ? "application/octet-stream" : part.type || "application/octet-stream";
1843
- args.push(parameter.schema._httpBody ? new HttpBody(bytes, partContentType) : new File(bytes, partContentType));
1844
- } else {
1845
- try {
1846
- const bytes = typeof part === "string" ? new TextEncoder().encode(part) : new Uint8Array(await part.arrayBuffer());
1847
- args.push(deserializeJSON(bytes));
1848
- } catch (error) {
1849
- if (error instanceof DeserializationError) throw error;
1850
- throw new DeserializationError(
1851
- `Failed to deserialize multipart argument '${parameter.name}' as JSON`,
1852
- { cause: error }
1853
- );
1854
- }
1855
- }
1856
- }
1857
- return { args };
1858
- }
1859
- if (data.byteLength === 0 && (!firstParameter?.schema._file || !hasExplicitContentType)) {
1860
- return { args: [] };
1861
- }
1862
- if (firstParameter?.schema._file) return { args: [new File(data, contentType)] };
1863
- if (!contentType.toLowerCase().includes("json")) {
1864
- throw new DeserializationError(
1865
- `Expected application/json input for '${firstParameter?.name ?? "argument"}', got ${contentType}`
1866
- );
1867
- }
1868
- return { args: [deserializeJSON(data)] };
1869
- }
1870
- var PASSIVE_ASYNC_RESOURCE_TYPES = /* @__PURE__ */ new Set([
1871
- "PIPEWRAP",
1872
- "PIPESERVERWRAP",
1873
- "SIGNALWRAP",
1874
- "TCPSERVERWRAP",
1875
- "TCPWRAP",
1876
- "TLSWRAP",
1877
- "TTYWRAP",
1878
- "UDPWRAP"
1879
- ]);
1880
- var ReplayCausality = class {
1881
- constructor(onQuiescent) {
1882
- this.onQuiescent = onQuiescent;
1883
- this.hook = (0, import_node_async_hooks3.createHook)({
1884
- init: (asyncId, type, triggerAsyncId, resource) => {
1885
- if (this.stopped) return;
1886
- const triggerPromise = this.resources.get(triggerAsyncId)?.promise?.deref();
1887
- const resumesDetachedDurable = type === "PROMISE" && triggerPromise != null && isDetachedDurableThenable(triggerPromise);
1888
- const children = this.children.get(triggerAsyncId) ?? /* @__PURE__ */ new Set();
1889
- children.add(asyncId);
1890
- this.children.set(triggerAsyncId, children);
1891
- if (type === "PROMISE" && typeof resource === "object" && resource != null) {
1892
- this.promiseIds.set(resource, asyncId);
1893
- }
1894
- const triggerOwners = new Set(this.owners.get(triggerAsyncId) ?? []);
1895
- this.resources.set(asyncId, {
1896
- type,
1897
- triggerAsyncId,
1898
- promise: type === "PROMISE" && typeof resource === "object" && resource != null ? new WeakRef(resource) : void 0,
1899
- pending: true
1900
- });
1901
- if (resumesDetachedDurable) {
1902
- this.nativeDurableAssimilations.add(asyncId);
1903
- this.promiseAggregateAssimilationScopes.at(-1)?.add(asyncId);
1904
- this.suspendDurableAssimilation(asyncId);
1905
- }
1906
- if (this.suppressOwnership) return;
1907
- const inherited = /* @__PURE__ */ new Set([
1908
- ...this.owners.get((0, import_node_async_hooks3.executionAsyncId)()) ?? [],
1909
- ...triggerOwners
1910
- ]);
1911
- for (const token of inherited) this.addTokenToSubtree(asyncId, token);
1912
- },
1913
- promiseResolve: (asyncId) => {
1914
- if (this.stopped) return;
1915
- this.notifyPromiseSettlement(asyncId);
1916
- if (this.nativeDurableAssimilations.delete(asyncId)) {
1917
- this.resumeDurableAssimilation(asyncId);
1918
- }
1919
- if (this.explicitDurableAssimilations.delete(asyncId)) {
1920
- this.resumeDurableAssimilation(asyncId);
1921
- }
1922
- this.settleMixedPromiseProducerGroups(asyncId);
1923
- const currentOwners = this.owners.get((0, import_node_async_hooks3.executionAsyncId)());
1924
- if (currentOwners != null) {
1925
- for (const token of currentOwners) this.addTokenToSubtree(asyncId, token);
1926
- }
1927
- this.setResourcePending(asyncId, false);
1928
- this.scheduleCheck();
1929
- },
1930
- destroy: (asyncId) => {
1931
- if (this.stopped) return;
1932
- this.notifyPromiseSettlement(asyncId);
1933
- if (this.nativeDurableAssimilations.delete(asyncId)) {
1934
- this.detachDurableAssimilation(asyncId);
1935
- }
1936
- if (this.explicitDurableAssimilations.delete(asyncId)) {
1937
- this.detachDurableAssimilation(asyncId);
1938
- }
1939
- this.settleMixedPromiseProducerGroups(asyncId);
1940
- this.setResourcePending(asyncId, false);
1941
- this.releaseDestroyedResource(asyncId);
1942
- this.scheduleCheck();
1943
- }
1944
- });
1945
- this.hook.enable();
1946
- }
1947
- hook;
1948
- children = /* @__PURE__ */ new Map();
1949
- resources = /* @__PURE__ */ new Map();
1950
- owners = /* @__PURE__ */ new Map();
1951
- provenance = /* @__PURE__ */ new Map();
1952
- tokenResources = /* @__PURE__ */ new Map();
1953
- tokenProducers = /* @__PURE__ */ new Map();
1954
- promiseIds = /* @__PURE__ */ new WeakMap();
1955
- excludedRoots = /* @__PURE__ */ new Set();
1956
- durableAssimilations = /* @__PURE__ */ new Map();
1957
- nativeDurableAssimilations = /* @__PURE__ */ new Set();
1958
- explicitDurableAssimilations = /* @__PURE__ */ new Set();
1959
- promiseAggregateAssimilationScopes = [];
1960
- promiseSettlementObservers = /* @__PURE__ */ new Map();
1961
- mixedPromiseProducerGroups = /* @__PURE__ */ new Set();
1962
- nextToken = 1;
1963
- checkScheduled = false;
1964
- checkHandle;
1965
- suppressOwnership = false;
1966
- stopped = false;
1967
- runRoot(callback) {
1968
- const resource = new import_node_async_hooks3.AsyncResource("TensorlakeReplayCausality");
1969
- let result;
1970
- resource.runInAsyncScope(() => {
1971
- this.beginCurrentContinuation();
1972
- result = callback();
1973
- this.excludePromise(result);
1974
- });
1975
- resource.emitDestroy();
1976
- return result;
1977
- }
1978
- beginCurrentContinuation() {
1979
- if (this.stopped) return;
1980
- const asyncId = (0, import_node_async_hooks3.executionAsyncId)();
1981
- this.excludedRoots.delete(asyncId);
1982
- this.addTokenToSubtree(asyncId, this.nextToken);
1983
- this.nextToken += 1;
1984
- this.scheduleCheck();
1985
- }
1986
- excludePromise(promise) {
1987
- const asyncId = this.promiseIds.get(promise);
1988
- if (asyncId == null) return;
1989
- this.excludedRoots.add(asyncId);
1990
- this.removeOwnersFromSubtree(asyncId);
1991
- this.scheduleCheck();
1992
- }
1993
- suspendDurableAssimilation(asyncId) {
1994
- if (this.stopped) return false;
1995
- const count = this.durableAssimilations.get(asyncId) ?? 0;
1996
- this.durableAssimilations.set(asyncId, count + 1);
1997
- if (count === 0) {
1998
- this.excludedRoots.add(asyncId);
1999
- this.removeOwnersFromSubtree(asyncId);
2000
- }
2001
- this.scheduleCheck();
2002
- return true;
2003
- }
2004
- resumeDurableAssimilation(asyncId) {
2005
- if (this.stopped) return;
2006
- const count = this.durableAssimilations.get(asyncId);
2007
- if (count == null) return;
2008
- if (count > 1) {
2009
- this.durableAssimilations.set(asyncId, count - 1);
2010
- return;
2011
- }
2012
- this.durableAssimilations.delete(asyncId);
2013
- this.excludedRoots.delete(asyncId);
2014
- const currentOwners = this.owners.get((0, import_node_async_hooks3.executionAsyncId)());
2015
- if (currentOwners != null) {
2016
- for (const token of currentOwners) this.addTokenToSubtree(asyncId, token);
2017
- }
2018
- }
2019
- detachDurableAssimilation(asyncId) {
2020
- if (this.stopped) return;
2021
- const count = this.durableAssimilations.get(asyncId);
2022
- if (count == null) return;
2023
- if (count > 1) {
2024
- this.durableAssimilations.set(asyncId, count - 1);
2025
- return;
2026
- }
2027
- this.durableAssimilations.delete(asyncId);
2028
- this.excludedRoots.delete(asyncId);
2029
- this.scheduleCheck();
2030
- }
2031
- hasPendingDurableAssimilation() {
2032
- return this.durableAssimilations.size > 0;
2033
- }
2034
- hasPendingProducer() {
2035
- if (this.mixedPromiseProducerGroups.size > 0) return true;
2036
- for (const producers of this.tokenProducers.values()) {
2037
- if (producers.size > 0) return true;
2038
- }
2039
- return false;
2040
- }
2041
- requestCheck() {
2042
- this.scheduleCheck();
2043
- }
2044
- trackMixedPromiseRace(promise, producers = [promise]) {
2045
- if (this.stopped) return;
2046
- const producerIds = /* @__PURE__ */ new Set();
2047
- for (const producer of producers) {
2048
- const asyncId = this.promiseIds.get(producer);
2049
- if (asyncId == null) continue;
2050
- if (this.resources.get(asyncId)?.pending !== true) {
2051
- this.scheduleCheck();
2052
- return;
2053
- }
2054
- producerIds.add(asyncId);
2055
- }
2056
- if (producerIds.size > 0) this.mixedPromiseProducerGroups.add(producerIds);
2057
- this.scheduleCheck();
2058
- }
2059
- trackPromiseSettlement(promise, onSettled) {
2060
- if (this.stopped) return false;
2061
- const asyncId = this.promiseIds.get(promise);
2062
- if (asyncId == null) return false;
2063
- if (this.resources.get(asyncId)?.pending !== true) {
2064
- onSettled();
2065
- return true;
2066
- }
2067
- const observers = this.promiseSettlementObservers.get(asyncId) ?? /* @__PURE__ */ new Set();
2068
- observers.add(onSettled);
2069
- this.promiseSettlementObservers.set(asyncId, observers);
2070
- return true;
2071
- }
2072
- trackDurablePromiseConsumption(promise) {
2073
- if (this.stopped) return;
2074
- const asyncId = this.promiseIds.get(promise);
2075
- if (asyncId == null || this.resources.get(asyncId)?.pending !== true || this.nativeDurableAssimilations.has(asyncId) || this.explicitDurableAssimilations.has(asyncId)) {
2076
- return;
2077
- }
2078
- this.explicitDurableAssimilations.add(asyncId);
2079
- this.promiseAggregateAssimilationScopes.at(-1)?.add(asyncId);
2080
- this.suspendDurableAssimilation(asyncId);
2081
- }
2082
- runPromiseAggregateConstruction(callback) {
2083
- const aggregateAssimilations = /* @__PURE__ */ new Set();
2084
- this.promiseAggregateAssimilationScopes.push(aggregateAssimilations);
2085
- try {
2086
- return callback();
2087
- } finally {
2088
- this.promiseAggregateAssimilationScopes.pop();
2089
- this.detachPromiseAggregateAssimilations(aggregateAssimilations);
2090
- }
2091
- }
2092
- runWithoutOwnership(callback) {
2093
- if (this.stopped) return callback();
2094
- const previous = this.suppressOwnership;
2095
- this.suppressOwnership = true;
2096
- try {
2097
- return callback();
2098
- } finally {
2099
- this.suppressOwnership = previous;
2100
- }
2101
- }
2102
- stop() {
2103
- if (this.stopped) return;
2104
- this.stopped = true;
2105
- this.hook.disable();
2106
- if (this.checkHandle != null) clearImmediate(this.checkHandle);
2107
- this.checkHandle = void 0;
2108
- this.children.clear();
2109
- this.resources.clear();
2110
- this.owners.clear();
2111
- this.provenance.clear();
2112
- this.tokenResources.clear();
2113
- this.tokenProducers.clear();
2114
- this.durableAssimilations.clear();
2115
- this.nativeDurableAssimilations.clear();
2116
- this.explicitDurableAssimilations.clear();
2117
- this.promiseAggregateAssimilationScopes.length = 0;
2118
- this.promiseSettlementObservers.clear();
2119
- this.mixedPromiseProducerGroups.clear();
2120
- }
2121
- notifyPromiseSettlement(asyncId) {
2122
- const observers = this.promiseSettlementObservers.get(asyncId);
2123
- if (observers == null) return;
2124
- this.promiseSettlementObservers.delete(asyncId);
2125
- for (const observer of observers) observer();
2126
- }
2127
- settleMixedPromiseProducerGroups(asyncId) {
2128
- for (const group of [...this.mixedPromiseProducerGroups]) {
2129
- if (group.has(asyncId)) this.mixedPromiseProducerGroups.delete(group);
2130
- }
2131
- }
2132
- detachPromiseAggregateAssimilations(assimilations) {
2133
- for (const asyncId of assimilations) {
2134
- if (this.nativeDurableAssimilations.delete(asyncId)) {
2135
- this.detachDurableAssimilation(asyncId);
2136
- }
2137
- if (this.explicitDurableAssimilations.delete(asyncId)) {
2138
- this.detachDurableAssimilation(asyncId);
2139
- }
2140
- }
2141
- assimilations.clear();
2142
- }
2143
- addTokenToSubtree(asyncId, token) {
2144
- if (this.excludedRoots.has(asyncId)) return;
2145
- const owners = this.owners.get(asyncId) ?? /* @__PURE__ */ new Set();
2146
- if (!owners.has(token)) {
2147
- owners.add(token);
2148
- this.owners.set(asyncId, owners);
2149
- const provenance = this.provenance.get(asyncId) ?? /* @__PURE__ */ new Set();
2150
- provenance.add(token);
2151
- this.provenance.set(asyncId, provenance);
2152
- const tokenResources = this.tokenResources.get(token) ?? /* @__PURE__ */ new Set();
2153
- tokenResources.add(asyncId);
2154
- this.tokenResources.set(token, tokenResources);
2155
- const resource = this.resources.get(asyncId);
2156
- if (resource?.pending && this.isProducer(resource, token)) {
2157
- const producers = this.tokenProducers.get(token) ?? /* @__PURE__ */ new Set();
2158
- producers.add(asyncId);
2159
- this.tokenProducers.set(token, producers);
2160
- }
2161
- }
2162
- for (const child of this.children.get(asyncId) ?? []) {
2163
- this.addTokenToSubtree(child, token);
2164
- }
2165
- }
2166
- isProducer(resource, token) {
2167
- if (resource.type === "PROMISE") {
2168
- return !(this.provenance.get(resource.triggerAsyncId)?.has(token) ?? false);
2169
- }
2170
- return !PASSIVE_ASYNC_RESOURCE_TYPES.has(resource.type);
2171
- }
2172
- setResourcePending(asyncId, pending) {
2173
- const resource = this.resources.get(asyncId);
2174
- if (resource == null || resource.pending === pending) return;
2175
- resource.pending = pending;
2176
- for (const token of this.owners.get(asyncId) ?? []) {
2177
- const producers = this.tokenProducers.get(token);
2178
- if (pending && this.isProducer(resource, token)) {
2179
- const next = producers ?? /* @__PURE__ */ new Set();
2180
- next.add(asyncId);
2181
- this.tokenProducers.set(token, next);
2182
- } else {
2183
- producers?.delete(asyncId);
2184
- }
2185
- }
2186
- }
2187
- removeOwnersFromSubtree(asyncId) {
2188
- this.removeResourceOwnership(asyncId);
2189
- for (const child of this.children.get(asyncId) ?? []) {
2190
- this.removeOwnersFromSubtree(child);
2191
- }
2192
- }
2193
- removeResourceOwnership(asyncId) {
2194
- const owners = this.owners.get(asyncId);
2195
- if (owners == null) return;
2196
- for (const token of owners) {
2197
- const resources = this.tokenResources.get(token);
2198
- resources?.delete(asyncId);
2199
- if (resources?.size === 0) this.tokenResources.delete(token);
2200
- const producers = this.tokenProducers.get(token);
2201
- producers?.delete(asyncId);
2202
- if (producers?.size === 0) this.tokenProducers.delete(token);
2203
- }
2204
- this.owners.delete(asyncId);
2205
- }
2206
- releaseDestroyedResource(asyncId) {
2207
- const resource = this.resources.get(asyncId);
2208
- if (resource == null) return;
2209
- this.resources.delete(asyncId);
2210
- this.excludedRoots.delete(asyncId);
2211
- this.removeResourceOwnership(asyncId);
2212
- this.provenance.delete(asyncId);
2213
- const siblings = this.children.get(resource.triggerAsyncId);
2214
- siblings?.delete(asyncId);
2215
- if (siblings?.size === 0) this.children.delete(resource.triggerAsyncId);
2216
- this.children.delete(asyncId);
2217
- }
2218
- scheduleCheck() {
2219
- if (this.stopped || this.checkScheduled) return;
2220
- this.checkScheduled = true;
2221
- const previous = this.suppressOwnership;
2222
- this.suppressOwnership = true;
2223
- try {
2224
- this.checkHandle = setImmediate(() => {
2225
- this.checkScheduled = false;
2226
- this.checkHandle = void 0;
2227
- this.onQuiescent();
2228
- });
2229
- } finally {
2230
- this.suppressOwnership = previous;
2231
- }
2232
- }
2233
- };
2234
- var ReplayAwarePromise = class extends Promise {
2235
- causality;
2236
- constructor(source, causality) {
2237
- let bridge;
2238
- super((resolve, reject) => {
2239
- bridge = source.then(resolve, reject);
2240
- });
2241
- this.causality = causality;
2242
- registerDurableThenable(this);
2243
- causality.excludePromise(source);
2244
- causality.excludePromise(this);
2245
- if (bridge != null) causality.excludePromise(bridge);
2246
- }
2247
- static get [Symbol.species]() {
2248
- return Promise;
2249
- }
2250
- then(onfulfilled, onrejected) {
2251
- const assimilationId = (0, import_node_async_hooks3.executionAsyncId)();
2252
- const assimilationSuspended = this.causality.suspendDurableAssimilation(assimilationId);
2253
- let assimilationDetached = false;
2254
- const aggregateAssimilation = registerDurableAssimilation(this, () => {
2255
- if (!assimilationSuspended || assimilationDetached) return;
2256
- assimilationDetached = true;
2257
- this.causality.detachDurableAssimilation(assimilationId);
2258
- });
2259
- const continuation = super.then(
2260
- (value) => {
2261
- this.causality.beginCurrentContinuation();
2262
- aggregateAssimilation.finish();
2263
- if (assimilationSuspended && !assimilationDetached) {
2264
- this.causality.resumeDurableAssimilation(assimilationId);
2265
- }
2266
- if (onfulfilled == null) return value;
2267
- return onfulfilled(value);
2268
- },
2269
- (reason) => {
2270
- this.causality.beginCurrentContinuation();
2271
- aggregateAssimilation.finish();
2272
- if (assimilationSuspended && !assimilationDetached) {
2273
- this.causality.resumeDurableAssimilation(assimilationId);
2274
- }
2275
- if (onrejected == null) throw reason;
2276
- return onrejected(reason);
2277
- }
2278
- );
2279
- aggregateAssimilation.link(continuation);
2280
- this.causality.excludePromise(continuation);
2281
- return continuation;
2282
- }
2283
- [DURABLE_PROMISE_INTERNAL_THEN](onfulfilled, onrejected) {
2284
- const continuation = super.then(onfulfilled, onrejected);
2285
- registerDurableThenable(continuation, this);
2286
- this.causality.excludePromise(continuation);
2287
- return continuation;
2288
- }
2289
- [DURABLE_PROMISE_OBSERVER](onfulfilled, onrejected) {
2290
- const observation = super.then(onfulfilled, onrejected);
2291
- this.causality.excludePromise(observation);
2292
- void observation.catch(() => void 0);
2293
- }
2294
- };
2295
- var ReplayHistory = class {
2296
- entries;
2297
- calls = [];
2298
- watchers = [];
2299
- availableResults = /* @__PURE__ */ new Map();
2300
- resultWaiters = /* @__PURE__ */ new Map();
2301
- endWaiters = [];
2302
- createdWatcherIds = /* @__PURE__ */ new Set();
2303
- releasedResultIds = /* @__PURE__ */ new Set();
2304
- expectedCallIds = /* @__PURE__ */ new Set();
2305
- expectedWatcherIds = /* @__PURE__ */ new Set();
2306
- causality;
2307
- // A replayed result may still be downloading before it can resume user code.
2308
- activeResultResolutions = 0;
2309
- callCursor = 0;
2310
- watcherCursor = 0;
2311
- prefixCursor = 0;
2312
- mismatch;
2313
- constructor(entries) {
2314
- this.causality = new ReplayCausality(() => this.failBlockedResultWithoutProducer());
2315
- this.entries = entries.map((entry) => {
2316
- if (entry.functionCallWatcherResult != null) {
2317
- return { kind: "result", event: entry.functionCallWatcherResult, consumed: false };
2318
- }
2319
- if (entry.functionCallCreated != null) {
2320
- const replayEntry = {
2321
- kind: "call",
2322
- event: entry.functionCallCreated,
2323
- consumed: false
2324
- };
2325
- this.calls.push(replayEntry);
2326
- return replayEntry;
2327
- }
2328
- if (entry.functionCallWatcherCreated != null) {
2329
- const replayEntry = {
2330
- kind: "watcher",
2331
- event: entry.functionCallWatcherCreated,
2332
- consumed: false
2333
- };
2334
- this.watchers.push(replayEntry);
2335
- return replayEntry;
2336
- }
2337
- return { kind: "unknown", event: entry, consumed: false };
2338
- });
2339
- }
2340
- async takeOrdered(kind, id) {
2341
- this.advancePrefix();
2342
- const entries = kind === "call" ? this.calls : this.watchers;
2343
- const cursor = kind === "call" ? this.callCursor : this.watcherCursor;
2344
- const expectedIds = kind === "call" ? this.expectedCallIds : this.expectedWatcherIds;
2345
- if (cursor >= entries.length) {
2346
- expectedIds.delete(id);
2347
- if (this.prefixCursor === this.entries.length) return void 0;
2348
- const blocked = this.entries[this.prefixCursor];
2349
- if (kind !== "call" || blocked.kind !== "watcher" || !this.expectedWatcherIds.has(String(blocked.event.functionCallId ?? ""))) {
2350
- this.fail(
2351
- `Unexpected ${kind} event for ${id} before replay ${blocked.kind} event ${String(blocked.event.functionCallId ?? "")}`
2352
- );
2353
- }
2354
- const waiter = deferred();
2355
- this.endWaiters.push(waiter);
2356
- await waiter.promise;
2357
- this.assertNoMismatch();
2358
- return void 0;
2359
- }
2360
- const entry = entries[cursor];
2361
- if (kind === "call") this.callCursor += 1;
2362
- else this.watcherCursor += 1;
2363
- if (entry.event.functionCallId !== id) {
2364
- this.fail(`Expected replay ${kind} event for ${id}`);
2365
- }
2366
- expectedIds.delete(id);
2367
- entry.consumed = true;
2368
- if (kind === "watcher") {
2369
- this.createdWatcherIds.add(id);
2370
- }
2371
- this.advancePrefix();
2372
- return entry.event;
2373
- }
2374
- async takeResult(id) {
2375
- this.advancePrefix();
2376
- const available = this.availableResults.get(id);
2377
- if (available != null && available.length > 0) {
2378
- const result2 = available.shift();
2379
- if (result2 != null) this.activeResultResolutions += 1;
2380
- return result2;
2381
- }
2382
- if (this.prefixCursor === this.entries.length) return void 0;
2383
- const blocked = this.entries[this.prefixCursor];
2384
- const blockedId = String(blocked.event.functionCallId ?? "");
2385
- const hasProducer = blocked.kind === "call" ? this.expectedCallIds.has(blockedId) : blocked.kind === "watcher" ? this.expectedWatcherIds.has(blockedId) : false;
2386
- const waiter = deferred();
2387
- const waiters = this.resultWaiters.get(id) ?? [];
2388
- waiters.push(waiter);
2389
- this.resultWaiters.set(id, waiters);
2390
- if (!hasProducer) this.causality.requestCheck();
2391
- const result = await waiter.promise;
2392
- if (result != null) this.activeResultResolutions += 1;
2393
- return result;
2394
- }
2395
- completeResultResolution() {
2396
- if (this.activeResultResolutions === 0) return;
2397
- this.activeResultResolutions -= 1;
2398
- this.causality.requestCheck();
2399
- }
2400
- runCausalHandler(callback) {
2401
- return this.causality.runRoot(callback);
2402
- }
2403
- wrapDurableResult(source) {
2404
- return new ReplayAwarePromise(source, this.causality);
2405
- }
2406
- trackMixedPromiseRace(promise, producers) {
2407
- this.causality.trackMixedPromiseRace(promise, producers);
2408
- }
2409
- trackPromiseSettlement(promise, onSettled) {
2410
- return this.causality.trackPromiseSettlement(promise, onSettled);
2411
- }
2412
- trackDurablePromiseConsumption(promise) {
2413
- this.causality.trackDurablePromiseConsumption(promise);
2414
- }
2415
- runPromiseAggregateConstruction(callback) {
2416
- return this.causality.runPromiseAggregateConstruction(callback);
2417
- }
2418
- runWithoutCausalOwnership(callback) {
2419
- return this.causality.runWithoutOwnership(callback);
2420
- }
2421
- dispose() {
2422
- this.causality.stop();
2423
- }
2424
- expectCall(id) {
2425
- this.expectedCallIds.add(id);
2426
- }
2427
- expectWatcher(id) {
2428
- this.expectedWatcherIds.add(id);
2429
- }
2430
- abandonCall(id) {
2431
- this.abandonOrdered("call", id);
2432
- }
2433
- abandonWatcher(id) {
2434
- this.abandonOrdered("watcher", id);
2435
- }
2436
- abandonOrdered(kind, id) {
2437
- const expectedIds = kind === "call" ? this.expectedCallIds : this.expectedWatcherIds;
2438
- expectedIds.delete(id);
2439
- const hasWaiters = this.endWaiters.length > 0 || [...this.resultWaiters.values()].some((waiters) => waiters.length > 0);
2440
- if (this.prefixCursor >= this.entries.length || !hasWaiters) return;
2441
- const blocked = this.entries[this.prefixCursor];
2442
- if (blocked.kind === kind && blocked.event.functionCallId === id) {
2443
- this.fail(`Replay ${kind} event for ${id} can no longer be created`);
2444
- }
2445
- }
2446
- failBlockedResultWithoutProducer() {
2447
- if (this.activeResultResolutions > 0) return;
2448
- if (!this.causality.hasPendingDurableAssimilation()) return;
2449
- if (this.causality.hasPendingProducer()) return;
2450
- if (this.prefixCursor >= this.entries.length) return;
2451
- if (![...this.resultWaiters.values()].some((waiters) => waiters.length > 0)) return;
2452
- const blocked = this.entries[this.prefixCursor];
2453
- const blockedId = String(blocked.event.functionCallId ?? "");
2454
- const hasProducer = blocked.kind === "call" ? this.expectedCallIds.has(blockedId) : blocked.kind === "watcher" ? this.expectedWatcherIds.has(blockedId) : false;
2455
- if (!hasProducer) {
2456
- this.recordMismatch(
2457
- `Replay result is blocked behind unreachable ${blocked.kind} event ${blockedId}`
2458
- );
2459
- }
2460
- }
2461
- hasAvailableResult(id) {
2462
- return (this.availableResults.get(id)?.length ?? 0) > 0;
2463
- }
2464
- assertNoMismatch() {
2465
- if (this.mismatch != null) throw this.mismatch;
2466
- }
2467
- assertConsumed() {
2468
- this.assertNoMismatch();
2469
- this.advancePrefix();
2470
- if (this.prefixCursor !== this.entries.length) {
2471
- throw new ReplayMismatchError("Function completed before its durable event history was consumed");
2472
- }
2473
- }
2474
- advancePrefix() {
2475
- this.assertNoMismatch();
2476
- while (this.prefixCursor < this.entries.length) {
2477
- const entry = this.entries[this.prefixCursor];
2478
- if (entry.kind === "unknown") {
2479
- this.fail("Replay history contains an unknown allocation event");
2480
- }
2481
- if (entry.kind === "call" || entry.kind === "watcher") {
2482
- if (!entry.consumed) return;
2483
- this.prefixCursor += 1;
2484
- continue;
2485
- }
2486
- entry.consumed = true;
2487
- this.prefixCursor += 1;
2488
- this.releaseResult(entry.event);
2489
- }
2490
- for (const waiter of this.endWaiters.splice(0)) waiter.resolve();
2491
- for (const waiters of this.resultWaiters.values()) {
2492
- for (const waiter of waiters) waiter.resolve(void 0);
2493
- }
2494
- this.resultWaiters.clear();
2495
- }
2496
- releaseResult(event) {
2497
- const id = String(event.functionCallId ?? "");
2498
- if (!this.createdWatcherIds.has(id)) {
2499
- this.fail(`Replay watcher result for ${id} appeared before its watcher was created`);
2500
- }
2501
- if (this.releasedResultIds.has(id)) {
2502
- this.fail(`Replay history contains duplicate watcher results for ${id}`);
2503
- }
2504
- this.releasedResultIds.add(id);
2505
- const waiter = this.resultWaiters.get(id)?.shift();
2506
- if (waiter != null) {
2507
- waiter.resolve(event);
2508
- if (this.resultWaiters.get(id)?.length === 0) this.resultWaiters.delete(id);
2509
- return;
2510
- }
2511
- const available = this.availableResults.get(id) ?? [];
2512
- available.push(event);
2513
- this.availableResults.set(id, available);
2514
- }
2515
- fail(message) {
2516
- throw this.recordMismatch(message);
2517
- }
2518
- recordMismatch(message) {
2519
- this.mismatch ??= new ReplayMismatchError(message);
2520
- this.causality.stop();
2521
- for (const waiter of this.endWaiters.splice(0)) waiter.reject(this.mismatch);
2522
- for (const waiters of this.resultWaiters.values()) {
2523
- for (const waiter of waiters) waiter.reject(this.mismatch);
2524
- }
2525
- this.resultWaiters.clear();
2526
- return this.mismatch;
2527
- }
2528
- };
2529
- var AllocationProtocolError = class extends Error {
2530
- constructor(message, options) {
2531
- super(message, options);
2532
- this.name = "AllocationProtocolError";
2533
- }
2534
- };
2535
- var AllocationRunner = class {
2536
- allocation;
2537
- functionRef;
2538
- definition;
2539
- applicationDefinition;
2540
- stateStreams = /* @__PURE__ */ new Set();
2541
- stateStreamFlows = /* @__PURE__ */ new Map();
2542
- eventReadStreams = /* @__PURE__ */ new Set();
2543
- outputBlobRequests = /* @__PURE__ */ new Map();
2544
- stateOperationRequests = /* @__PURE__ */ new Map();
2545
- executionBatches = [];
2546
- executionBatchWaiters = [];
2547
- liveBacklog = [];
2548
- liveWaiters = [];
2549
- controller = new AbortController();
2550
- currentRead;
2551
- replay;
2552
- protocolError;
2553
- durableCallEmissionTail = Promise.resolve();
2554
- durableWatcherEmissionTail = Promise.resolve();
2555
- lastEventClock = 0;
2556
- previousDurableId;
2557
- livePump;
2558
- completion;
2559
- started = false;
2560
- terminalBatchQueued = false;
2561
- terminalBatchObserved = false;
2562
- terminalBatchObservedWaiters = [];
2563
- finished = false;
2564
- state = { outputBlobRequests: [], requestStateOperations: [] };
2565
- startedAt = Date.now();
2566
- constructor(allocation, functionRef, definition, applicationDefinition = definition) {
2567
- this.allocation = allocation;
2568
- this.functionRef = functionRef;
2569
- this.definition = definition;
2570
- this.applicationDefinition = applicationDefinition;
2571
- this.previousDurableId = String(allocation.functionCallId);
2572
- this.updateStateHash();
2573
- this.log("debug", "allocation runner created", {
2574
- replay_mode: allocation.replayMode,
2575
- input_count: allocation.inputs?.args?.length ?? 0,
2576
- input_blob_count: allocation.inputs?.argBlobs?.length ?? 0
2577
- });
2578
- }
2579
- start() {
2580
- if (this.started) {
2581
- this.log("warn", "duplicate allocation runner start ignored");
2582
- return;
2583
- }
2584
- this.started = true;
2585
- this.log("info", "allocation runner scheduled");
2586
- this.completion = this.run().catch((error) => {
2587
- this.log("error", "allocation runner promise rejected", {}, error);
2588
- this.queueTerminalBatch({
2589
- outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
2590
- failureReason: "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR"
2591
- });
2592
- this.finish();
2593
- });
2594
- }
2595
- waitForCompletion() {
2596
- return this.completion ?? Promise.resolve();
2597
- }
2598
- async waitForTerminalBatchObserved(timeoutMs) {
2599
- if (this.terminalBatchObserved) return true;
2600
- return new Promise((resolve) => {
2601
- let settled = false;
2602
- const observed = () => {
2603
- if (settled) return;
2604
- settled = true;
2605
- clearTimeout(timer);
2606
- resolve(true);
2607
- };
2608
- const timer = setTimeout(() => {
2609
- if (settled) return;
2610
- settled = true;
2611
- const index = this.terminalBatchObservedWaiters.indexOf(observed);
2612
- if (index >= 0) this.terminalBatchObservedWaiters.splice(index, 1);
2613
- resolve(false);
2614
- }, Math.max(timeoutMs, 0));
2615
- this.terminalBatchObservedWaiters.push(observed);
2616
- if (this.terminalBatchObserved) observed();
2617
- });
2618
- }
2619
- cancel(reason = new FunctionError("Allocation was cancelled")) {
2620
- if (this.controller.signal.aborted) return;
2621
- this.log("info", "allocation cancellation requested", {
2622
- reason: reason instanceof Error ? reason.message : String(reason)
2623
- });
2624
- this.abortExecution(reason);
2625
- }
2626
- abortExecution(reason) {
2627
- if (!this.controller.signal.aborted) this.controller.abort(reason);
2628
- const rejection = this.controller.signal.reason ?? reason;
2629
- const outputBlobRequestCount = this.outputBlobRequests.size;
2630
- const stateOperationRequestCount = this.stateOperationRequests.size;
2631
- const liveWaiterCount = this.liveWaiters.length;
2632
- const liveBacklogCount = this.liveBacklog.length;
2633
- const hadCurrentRead = this.currentRead != null;
2634
- for (const request of this.outputBlobRequests.values()) request.reject(rejection);
2635
- for (const request of this.stateOperationRequests.values()) request.reject(rejection);
2636
- for (const waiter of this.liveWaiters.splice(0)) waiter.result.reject(rejection);
2637
- this.currentRead?.response.reject(rejection);
2638
- this.outputBlobRequests.clear();
2639
- this.stateOperationRequests.clear();
2640
- this.liveBacklog.splice(0);
2641
- this.currentRead = void 0;
2642
- this.state.outputBlobRequests = [];
2643
- this.state.requestStateOperations = [];
2644
- this.log("debug", "pending allocation protocol work settled", {
2645
- output_blob_requests: outputBlobRequestCount,
2646
- state_operation_requests: stateOperationRequestCount,
2647
- durable_event_waiters: liveWaiterCount,
2648
- durable_event_backlog_entries: liveBacklogCount,
2649
- event_log_read: hadCurrentRead
2650
- });
2651
- }
2652
- invoke(definition, args) {
2653
- this.log("debug", "durable function invocation requested", {
2654
- target_function: definition.name,
2655
- argument_count: args.length
2656
- });
2657
- const result = this.runChild(definition, args, 0);
2658
- return this.replay?.wrapDurableResult(result) ?? result;
2659
- }
2660
- runFuture(future) {
2661
- this.log("debug", "durable future execution requested", {
2662
- target_function: future.definition.name,
2663
- argument_count: future.args.length,
2664
- delay_seconds: future.delaySeconds
2665
- });
2666
- const result = this.runChild(future.definition, future.args, future.delaySeconds);
2667
- return this.replay?.wrapDurableResult(result) ?? result;
2668
- }
2669
- wrapFutureWait(promise) {
2670
- return this.replay?.wrapDurableResult(promise) ?? promise;
2671
- }
2672
- trackMixedPromiseRace(promise, producers) {
2673
- this.replay?.trackMixedPromiseRace(promise, producers);
2674
- }
2675
- trackPromiseSettlement(promise, onSettled) {
2676
- return this.replay?.trackPromiseSettlement(promise, onSettled) ?? false;
2677
- }
2678
- trackDurablePromiseConsumption(promise) {
2679
- this.replay?.trackDurablePromiseConsumption(promise);
2680
- }
2681
- runPromiseAggregateConstruction(callback) {
2682
- return this.replay?.runPromiseAggregateConstruction(callback) ?? callback();
2683
- }
2684
- reduce(definition, items, initial, hasInitial) {
2685
- const result = this.runReduce(definition, items, initial, hasInitial);
2686
- return this.replay?.wrapDurableResult(result) ?? result;
2687
- }
2688
- async runReduce(definition, items, initial, hasInitial) {
2689
- let reduceItems = items;
2690
- let reduceInitial = initial;
2691
- if (!hasInitial) {
2692
- if (items.length === 0) {
2693
- throw new SDKUsageError("reduce of empty iterable with no initial value");
2694
- }
2695
- [reduceInitial, ...reduceItems] = items;
2696
- }
2697
- if (reduceItems.length === 0) return reduceInitial;
2698
- const callTurn = this.reserveEmissionTurn("call");
2699
- const watcherTurn = this.reserveEmissionTurn("watcher");
2700
- const ids = reduceItems.map(() => this.nextDurableId(definition, "ReduceStep"));
2701
- const id = ids[ids.length - 1];
2702
- const startedAt = Date.now();
2703
- this.log("info", "durable reduce starting", {
2704
- durable_id: id,
2705
- target_function: definition.name,
2706
- collection_size: reduceItems.length,
2707
- plan_count: Math.ceil(reduceItems.length / MAX_REDUCE_CALLS_PER_PLAN)
2708
- });
2709
- const replayCallIds = ids.filter(
2710
- (_id, index) => (index + 1) % MAX_REDUCE_CALLS_PER_PLAN === 0 || index === ids.length - 1
2711
- );
2712
- for (const replayCallId of replayCallIds) this.replay?.expectCall(replayCallId);
2713
- this.replay?.expectWatcher(id);
2714
- try {
2715
- try {
2716
- await this.createReduceChain(ids, definition, reduceItems, reduceInitial, callTurn);
2717
- } catch (error) {
2718
- for (const replayCallId of replayCallIds) this.replay?.abandonCall(replayCallId);
2719
- this.replay?.abandonWatcher(id);
2720
- throw error;
2721
- }
2722
- await this.createWatcher(id, watcherTurn);
2723
- } finally {
2724
- callTurn.release();
2725
- watcherTurn.release();
2726
- }
2727
- const replayedResult = this.replay == null ? void 0 : await this.replay.takeResult(id);
2728
- const result = replayedResult ?? await this.waitForLiveEvent(
2729
- (entry) => entry.functionCallWatcherResult?.functionCallId === id
2730
- ).then((entry) => entry.functionCallWatcherResult);
2731
- let resolved;
2732
- try {
2733
- resolved = await this.resolveWatcherResult(result);
2734
- } finally {
2735
- if (replayedResult != null) this.replay?.completeResultResolution();
2736
- }
2737
- this.log("info", "durable reduce completed", {
2738
- durable_id: id,
2739
- target_function: definition.name,
2740
- collection_size: reduceItems.length,
2741
- duration_ms: Date.now() - startedAt
2742
- });
2743
- return resolved;
2744
- }
2745
- nextDurableId(definition, operation = "FunctionCall") {
2746
- const id = durableHash([
2747
- String(this.allocation.functionCallId),
2748
- this.previousDurableId,
2749
- operation,
2750
- definition.name
2751
- ]);
2752
- this.previousDurableId = id;
2753
- return id;
2754
- }
2755
- async runChild(definition, args, delaySeconds) {
2756
- const callTurn = this.reserveEmissionTurn("call");
2757
- const watcherTurn = this.reserveEmissionTurn("watcher");
2758
- const id = this.nextDurableId(definition);
2759
- const startedAt = Date.now();
2760
- const maxRetries = definition.options.retries?.maxRetries ?? this.applicationDefinition.application?.retries.maxRetries ?? 0;
2761
- this.log("info", "durable function call starting", {
2762
- durable_id: id,
2763
- target_function: definition.name,
2764
- argument_count: args.length,
2765
- delay_seconds: delaySeconds,
2766
- max_retries: maxRetries
2767
- });
2768
- this.replay?.expectCall(id);
2769
- this.replay?.expectWatcher(id);
2770
- try {
2771
- try {
2772
- await this.createChildCall(id, definition, args, delaySeconds, callTurn);
2773
- } catch (error) {
2774
- this.replay?.abandonCall(id);
2775
- this.replay?.abandonWatcher(id);
2776
- throw error;
2777
- }
2778
- await this.createWatcher(id, watcherTurn);
2779
- } finally {
2780
- callTurn.release();
2781
- watcherTurn.release();
2782
- }
2783
- this.log("debug", "waiting for terminal durable function result", {
2784
- durable_id: id,
2785
- target_function: definition.name,
2786
- max_retries: maxRetries,
2787
- replay_result_available: this.replay?.hasAvailableResult(id) ?? false
2788
- });
2789
- const replayedResult = this.replay == null ? void 0 : await this.replay.takeResult(id);
2790
- const result = replayedResult ?? await this.waitForLiveEvent(
2791
- (entry) => entry.functionCallWatcherResult?.functionCallId === id
2792
- ).then((entry) => entry.functionCallWatcherResult);
2793
- let resolved;
2794
- try {
2795
- resolved = await this.resolveWatcherResult(result);
2796
- } finally {
2797
- if (replayedResult != null) this.replay?.completeResultResolution();
2798
- }
2799
- this.log("info", "durable function call completed", {
2800
- durable_id: id,
2801
- target_function: definition.name,
2802
- max_retries: maxRetries,
2803
- duration_ms: Date.now() - startedAt
2804
- });
2805
- return resolved;
2806
- }
2807
- async createChildCall(id, definition, args, delaySeconds, turn) {
2808
- try {
2809
- let turnAcquired = false;
2810
- if (this.replay != null) {
2811
- await turn.wait;
2812
- turnAcquired = true;
2813
- }
2814
- const replayed = await this.replay?.takeOrdered("call", id);
2815
- if (replayed != null) {
2816
- this.log("debug", "replaying durable function creation", { durable_id: id });
2817
- const error2 = statusError(replayed.status, "Failed to start function call");
2818
- if (error2 != null) throw error2;
2819
- return;
2820
- }
2821
- const prepared = [];
2822
- let offset = 0;
2823
- for (const arg of args) {
2824
- const item = prepareSerializedObject(arg, offset);
2825
- prepared.push(item);
2826
- offset += item.bytes.byteLength;
2827
- }
2828
- let argsBlob;
2829
- if (prepared.length > 0) {
2830
- this.log("debug", "requesting durable function arguments blob", {
2831
- durable_id: id,
2832
- argument_count: args.length,
2833
- serialized_bytes: offset
2834
- });
2835
- try {
2836
- argsBlob = await this.requestOutputBlob(offset);
2837
- argsBlob = await this.uploadBlob(
2838
- argsBlob,
2839
- joinPrepared(prepared),
2840
- "durable function arguments",
2841
- { durable_id: id }
2842
- );
2843
- } catch (error2) {
2844
- if (this.controller.signal.aborted) throw this.controller.signal.reason;
2845
- throw this.latchProtocolError(
2846
- `Failed to materialize durable function arguments for '${id}'`,
2847
- error2
2848
- );
2849
- }
2850
- }
2851
- const metadata = Buffer.from(JSON.stringify({
2852
- format: "tensorlake.typescript.function-call.v1",
2853
- id,
2854
- functionName: definition.name,
2855
- argumentCount: args.length
2856
- }), "utf8");
2857
- const updates = {
2858
- updates: [{ functionCall: {
2859
- id,
2860
- target: {
2861
- namespace: this.functionRef.namespace,
2862
- applicationName: this.functionRef.applicationName,
2863
- applicationVersion: this.functionRef.applicationVersion,
2864
- functionName: definition.name
2865
- },
2866
- args: prepared.map((item) => ({ value: item.object })),
2867
- callMetadata: metadata
2868
- } }],
2869
- rootFunctionCallId: id
2870
- };
2871
- if (delaySeconds > 0) updates.startAt = protocolTimestamp(new Date(Date.now() + delaySeconds * 1e3));
2872
- if (!turnAcquired) await turn.wait;
2873
- this.log("debug", "queueing durable function creation event", {
2874
- durable_id: id,
2875
- target_function: definition.name,
2876
- delay_seconds: delaySeconds
2877
- });
2878
- this.addBatch([{ createFunctionCall: { updates, argsBlob } }]);
2879
- this.log("debug", "waiting for durable function creation acknowledgement", { durable_id: id });
2880
- const createdPromise = this.waitForLiveEvent((entry) => entry.functionCallCreated?.functionCallId === id);
2881
- turn.release();
2882
- const created = await createdPromise;
2883
- const error = statusError(created.functionCallCreated?.status, "Failed to start function call");
2884
- if (error != null) throw error;
2885
- } finally {
2886
- turn.release();
2887
- }
2888
- }
2889
- async createReduceChain(ids, definition, items, initial, turn) {
2890
- try {
2891
- let turnAcquired = false;
2892
- if (this.replay != null) {
2893
- await turn.wait;
2894
- turnAcquired = true;
2895
- }
2896
- const livePlans = [];
2897
- const liveRoots = [];
2898
- for (let start = 0; start < items.length; start += MAX_REDUCE_CALLS_PER_PLAN) {
2899
- const end = Math.min(items.length, start + MAX_REDUCE_CALLS_PER_PLAN);
2900
- const rootId = ids[end - 1];
2901
- const replayed = await this.replay?.takeOrdered("call", rootId);
2902
- if (replayed != null) {
2903
- this.log("debug", "replaying durable reduce plan creation", {
2904
- durable_id: rootId,
2905
- reduce_step_start: start,
2906
- reduce_step_end: end
2907
- });
2908
- const error = statusError(replayed.status, "Failed to start reduce operation");
2909
- if (error != null) throw error;
2910
- continue;
2911
- }
2912
- const prepared = [];
2913
- let offset = 0;
2914
- if (start === 0) {
2915
- const value = prepareSerializedObject(initial, offset);
2916
- prepared.push(value);
2917
- offset += value.bytes.byteLength;
2918
- }
2919
- for (let index = start; index < end; index += 1) {
2920
- const value = prepareSerializedObject(items[index], offset);
2921
- prepared.push(value);
2922
- offset += value.bytes.byteLength;
2923
- }
2924
- let argsBlob;
2925
- try {
2926
- const requested = await this.requestOutputBlob(offset);
2927
- argsBlob = await this.uploadBlob(
2928
- requested,
2929
- joinPrepared(prepared),
2930
- "durable reduce arguments",
2931
- { durable_id: rootId, reduce_step_start: start, reduce_step_end: end }
2932
- );
2933
- } catch (error) {
2934
- if (this.controller.signal.aborted) throw this.controller.signal.reason;
2935
- throw this.latchProtocolError(
2936
- `Failed to materialize durable reduce arguments for '${rootId}'`,
2937
- error
2938
- );
2939
- }
2940
- const itemOffset = start === 0 ? 1 : 0;
2941
- const updates = [];
2942
- for (let index = start; index < end; index += 1) {
2943
- const id = ids[index];
2944
- const accumulator = index === 0 ? { value: prepared[0].object } : { functionCallId: ids[index - 1] };
2945
- const item = prepared[itemOffset + index - start];
2946
- const metadata = Buffer.from(JSON.stringify({
2947
- format: "tensorlake.typescript.function-call.v1",
2948
- id,
2949
- functionName: definition.name,
2950
- argumentCount: 2,
2951
- operation: "reduce",
2952
- reduceRootId: ids[ids.length - 1],
2953
- reduceStep: index,
2954
- reduceStepCount: items.length
2955
- }), "utf8");
2956
- updates.push({ functionCall: {
2957
- id,
2958
- target: {
2959
- namespace: this.functionRef.namespace,
2960
- applicationName: this.functionRef.applicationName,
2961
- applicationVersion: this.functionRef.applicationVersion,
2962
- functionName: definition.name
2963
- },
2964
- args: [accumulator, { value: item.object }],
2965
- callMetadata: metadata
2966
- } });
2967
- }
2968
- livePlans.push({
2969
- createFunctionCall: {
2970
- updates: { updates, rootFunctionCallId: rootId },
2971
- argsBlob
2972
- }
2973
- });
2974
- liveRoots.push(rootId);
2975
- }
2976
- if (livePlans.length === 0) return;
2977
- if (!turnAcquired) await turn.wait;
2978
- this.log("debug", "queueing durable reduce function-call chains", {
2979
- durable_id: ids[ids.length - 1],
2980
- target_function: definition.name,
2981
- reduce_step_count: items.length,
2982
- plan_count: livePlans.length
2983
- });
2984
- this.addBatch(livePlans);
2985
- const creationWaiters = liveRoots.map((rootId) => {
2986
- const predicate = (entry) => entry.functionCallCreated?.functionCallId === rootId;
2987
- return {
2988
- predicate,
2989
- promise: this.waitForLiveEvent(predicate)
2990
- };
2991
- });
2992
- const createdPromises = creationWaiters.map(async ({ promise }) => {
2993
- const created = await promise;
2994
- const error = statusError(
2995
- created.functionCallCreated?.status,
2996
- "Failed to start reduce operation"
2997
- );
2998
- if (error != null) throw error;
2999
- });
3000
- turn.release();
3001
- try {
3002
- await Promise.all(createdPromises);
3003
- } catch (error) {
3004
- for (const { predicate } of creationWaiters) {
3005
- const index = this.liveWaiters.findIndex(
3006
- (waiter) => waiter.predicate === predicate
3007
- );
3008
- if (index >= 0) this.liveWaiters.splice(index, 1)[0].result.reject(error);
3009
- }
3010
- throw error;
3011
- }
3012
- } finally {
3013
- turn.release();
3014
- }
3015
- }
3016
- async createWatcher(id, turn) {
3017
- try {
3018
- await turn.wait;
3019
- const replayed = await this.replay?.takeOrdered("watcher", id);
3020
- if (replayed != null) {
3021
- this.log("debug", "replaying durable function watcher creation", { durable_id: id });
3022
- const error2 = statusError(replayed.status, "Failed to create function call watcher");
3023
- if (error2 != null) throw error2;
3024
- return;
3025
- }
3026
- this.log("debug", "queueing durable function watcher event", { durable_id: id });
3027
- this.addBatch([{ createFunctionCallWatcher: { functionCallId: id } }]);
3028
- this.log("debug", "waiting for durable function watcher acknowledgement", { durable_id: id });
3029
- const createdPromise = this.waitForLiveEvent(
3030
- (entry) => entry.functionCallWatcherCreated?.functionCallId === id
3031
- );
3032
- turn.release();
3033
- const created = await createdPromise;
3034
- const error = statusError(created.functionCallWatcherCreated?.status, "Failed to create function call watcher");
3035
- if (error != null) throw error;
3036
- } finally {
3037
- turn.release();
3038
- }
3039
- }
3040
- async resolveWatcherResult(event) {
3041
- this.log("debug", "resolving durable function watcher result", {
3042
- durable_id: event.functionCallId,
3043
- watcher_status: event.watcherStatus,
3044
- outcome_code: event.outcomeCode,
3045
- has_value_output: event.valueOutput != null,
3046
- has_request_error_output: event.requestErrorOutput != null
3047
- });
3048
- if (WATCHER_TIMED_OUT.has(event.watcherStatus)) throw new TimeoutError();
3049
- if (OUTCOME_SUCCESS.has(event.outcomeCode)) {
3050
- try {
3051
- const downloaded = await this.downloadSerializedObject(
3052
- event.valueOutput,
3053
- event.valueBlob,
3054
- "durable function result",
3055
- { durable_id: event.functionCallId }
3056
- );
3057
- return deserializeValueFromProtocol(downloaded);
3058
- } catch (error) {
3059
- if (this.controller.signal.aborted) throw this.controller.signal.reason;
3060
- throw this.latchProtocolError(
3061
- `Invalid successful function call watcher payload for '${String(event.functionCallId)}'`,
3062
- error
3063
- );
3064
- }
3065
- }
3066
- if (OUTCOME_FAILURE.has(event.outcomeCode)) {
3067
- if (event.requestErrorOutput != null) {
3068
- let message;
3069
- try {
3070
- const downloaded = await this.downloadSerializedObject(
3071
- event.requestErrorOutput,
3072
- event.requestErrorBlob,
3073
- "durable function request error",
3074
- { durable_id: event.functionCallId }
3075
- );
3076
- message = new TextDecoder("utf-8", { fatal: true }).decode(downloaded.data);
3077
- } catch (error) {
3078
- if (this.controller.signal.aborted) throw this.controller.signal.reason;
3079
- throw this.latchProtocolError(
3080
- `Invalid request-error function call watcher payload for '${String(event.functionCallId)}'`,
3081
- error
3082
- );
3083
- }
3084
- throw new RequestError(message);
3085
- }
3086
- throw new FunctionError("Function call failed");
3087
- }
3088
- throw this.latchProtocolError(
3089
- `Unexpected function call watcher outcome '${String(event.outcomeCode)}'`
3090
- );
3091
- }
3092
- latchProtocolError(message, cause) {
3093
- const protocolError = this.protocolError ?? (message instanceof AllocationProtocolError ? message : new AllocationProtocolError(
3094
- message,
3095
- cause === void 0 ? void 0 : { cause }
3096
- ));
3097
- this.protocolError = protocolError;
3098
- this.abortExecution(protocolError);
3099
- return protocolError;
3100
- }
3101
- async startTailCall(tailCall) {
3102
- const callTurn = this.reserveEmissionTurn("call");
3103
- const id = this.nextDurableId(tailCall.definition);
3104
- this.log("info", "tail call starting", {
3105
- durable_id: id,
3106
- target_function: tailCall.definition.name,
3107
- argument_count: tailCall.args.length
3108
- });
3109
- this.replay?.expectCall(id);
3110
- try {
3111
- await this.createChildCall(id, tailCall.definition, tailCall.args, 0, callTurn);
3112
- } catch (error) {
3113
- this.replay?.abandonCall(id);
3114
- throw error;
3115
- }
3116
- return id;
3117
- }
3118
- reserveEmissionTurn(kind) {
3119
- const previous = kind === "call" ? this.durableCallEmissionTail : this.durableWatcherEmissionTail;
3120
- let signalRelease;
3121
- const released = new Promise((resolve) => {
3122
- signalRelease = resolve;
3123
- });
3124
- const tail = previous.then(() => released);
3125
- if (kind === "call") this.durableCallEmissionTail = tail;
3126
- else this.durableWatcherEmissionTail = tail;
3127
- let didRelease = false;
3128
- return {
3129
- wait: previous,
3130
- release: () => {
3131
- if (didRelease) return;
3132
- didRelease = true;
3133
- signalRelease();
3134
- }
3135
- };
3136
- }
3137
- async run() {
3138
- let handlerArgs;
3139
- this.log("info", "allocation execution started", {
3140
- replay_mode: this.allocation.replayMode,
3141
- function: this.functionRef.functionName
3142
- });
3143
- this.emitLifecycleEvent("allocations_started", "Starting allocations");
3144
- try {
3145
- const inputs = this.allocation.inputs ?? {};
3146
- const objects = inputs.args ?? [];
3147
- const blobs = inputs.argBlobs ?? [];
3148
- this.log("debug", "allocation inputs validated", {
3149
- object_count: objects.length,
3150
- blob_count: blobs.length,
3151
- function_call_metadata_bytes: inputs.functionCallMetadata?.length ?? 0
3152
- });
3153
- if (objects.length !== blobs.length) throw new Error("Function argument and BLOB counts differ");
3154
- const values = await Promise.all(objects.map(
3155
- (object, index) => this.downloadSerializedObject(
3156
- object,
3157
- blobs[index],
3158
- "function input",
3159
- { input_index: index }
3160
- )
3161
- ));
3162
- this.log("info", "all function inputs downloaded", {
3163
- input_count: values.length,
3164
- total_data_bytes: values.reduce((total, value) => total + value.data.byteLength, 0)
3165
- });
3166
- const metadata = Buffer.from(inputs.functionCallMetadata ?? []);
3167
- if (metadata.byteLength === 0) {
3168
- this.log("debug", "deserializing application invocation input", {
3169
- content_type: values[0]?.contentType
3170
- });
3171
- if (values.length !== 1) throw new Error("Application calls require exactly one input payload");
3172
- try {
3173
- const parsed = await deserializeApplicationArguments(this.definition, values[0]);
3174
- handlerArgs = parsed.args;
3175
- } catch (error) {
3176
- this.log("error", "application invocation input deserialization failed", {}, error);
3177
- await this.finishUserError(error);
3178
- return;
3179
- }
3180
- this.log("debug", "application invocation input deserialized", {
3181
- handler_argument_count: handlerArgs.length
3182
- });
3183
- } else {
3184
- this.log("debug", "deserializing internal function invocation inputs", {
3185
- metadata_bytes: metadata.byteLength
3186
- });
3187
- const call = JSON.parse(metadata.toString("utf8"));
3188
- if (call.format !== "tensorlake.typescript.function-call.v1") {
3189
- throw new Error("Function call metadata was produced by a different SDK language");
3190
- }
3191
- if (call.argumentCount !== values.length) throw new Error("Function call metadata argument count differs");
3192
- try {
3193
- handlerArgs = values.map(deserializeValueFromProtocol);
3194
- } catch (error) {
3195
- this.log("error", "internal function argument deserialization failed", {}, error);
3196
- await this.finishUserError(error);
3197
- return;
3198
- }
3199
- this.log("debug", "internal function invocation inputs deserialized", {
3200
- handler_argument_count: handlerArgs.length,
3201
- metadata_function: call.functionName
3202
- });
3203
- }
3204
- if (REPLAY_STRICT.has(this.allocation.replayMode)) {
3205
- this.log("info", "strict replay history loading");
3206
- await this.loadReplayHistory();
3207
- } else {
3208
- this.log("debug", "strict replay disabled");
3209
- }
3210
- const requestContext = this.createRequestContext();
3211
- let result;
3212
- const handlerStartedAt = Date.now();
3213
- this.log("info", "user function handler starting", {
3214
- handler_argument_count: handlerArgs.length
3215
- });
3216
- try {
3217
- const execute = () => executeHandlerResult(this.definition, handlerArgs);
3218
- result = await runWithRequestContext(
3219
- requestContext,
3220
- () => runWithFunctionRuntime(this, () => waitWithAbortSignal(
3221
- this.replay?.runCausalHandler(execute) ?? execute(),
3222
- this.controller.signal
3223
- ))
3224
- );
3225
- } catch (error) {
3226
- if (error instanceof ReplayMismatchError || error instanceof AllocationProtocolError) {
3227
- throw error;
3228
- }
3229
- this.replay?.assertNoMismatch();
3230
- this.log("error", "user function handler failed", {
3231
- duration_ms: Date.now() - handlerStartedAt
3232
- }, error);
3233
- await this.finishUserError(error);
3234
- return;
3235
- }
3236
- this.log("info", "user function handler completed", {
3237
- duration_ms: Date.now() - handlerStartedAt,
3238
- result_kind: isTailCall(result) ? "tail_call" : "value",
3239
- result_type: result == null ? String(result) : typeof result
3240
- });
3241
- if (this.protocolError != null) throw this.protocolError;
3242
- this.replay?.assertNoMismatch();
3243
- if (isTailCall(result)) {
3244
- const id = await this.startTailCall(result);
3245
- this.replay?.assertConsumed();
3246
- this.controller.signal.throwIfAborted();
3247
- this.log("info", "queueing successful tail-call allocation finish", { tail_call_durable_id: id });
3248
- this.queueTerminalBatch({
3249
- outcomeCode: "ALLOCATION_OUTCOME_CODE_SUCCESS",
3250
- tailCallDurableId: id
3251
- });
3252
- } else {
3253
- this.replay?.assertConsumed();
3254
- let prepared;
3255
- try {
3256
- prepared = prepareSerializedObject(result, 0, String(this.allocation.functionCallId));
3257
- } catch (error) {
3258
- this.log("error", "function output serialization failed", {}, error);
3259
- await this.finishUserError(error);
3260
- return;
3261
- }
3262
- this.log("debug", "function output serialized", {
3263
- output_bytes: prepared.bytes.byteLength,
3264
- output_encoding: prepared.object.manifest?.encoding,
3265
- output_content_type: prepared.object.manifest?.contentType
3266
- });
3267
- let outputBlob = await this.requestOutputBlob(prepared.bytes.byteLength);
3268
- outputBlob = await this.uploadBlob(outputBlob, prepared.bytes, "function output");
3269
- this.controller.signal.throwIfAborted();
3270
- this.log("info", "queueing successful value allocation finish", {
3271
- output_blob_id: outputBlob.id,
3272
- output_bytes: prepared.bytes.byteLength
3273
- });
3274
- this.queueTerminalBatch({
3275
- outcomeCode: "ALLOCATION_OUTCOME_CODE_SUCCESS",
3276
- value: prepared.object,
3277
- uploadedFunctionOutputsBlob: outputBlob
3278
- });
3279
- }
3280
- } catch (error) {
3281
- const replayMismatch = error instanceof ReplayMismatchError;
3282
- const protocolFailure = error instanceof AllocationProtocolError || this.protocolError != null;
3283
- const cancelled = this.controller.signal.aborted && !protocolFailure;
3284
- this.log(
3285
- cancelled ? "info" : "error",
3286
- cancelled ? "allocation stopped after cancellation" : replayMismatch ? "durable replay mismatch" : "allocation internal failure",
3287
- {},
3288
- error
3289
- );
3290
- this.queueTerminalBatch({
3291
- outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
3292
- failureReason: replayMismatch ? "ALLOCATION_FAILURE_REASON_REPLAY_EVENT_HISTORY_MISMATCH" : protocolFailure ? "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR" : cancelled ? "ALLOCATION_FAILURE_REASON_FUNCTION_ERROR" : "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR"
3293
- });
3294
- } finally {
3295
- this.log("info", "allocation execution stopping", {
3296
- duration_ms: Date.now() - this.startedAt,
3297
- pending_output_blob_requests: this.outputBlobRequests.size,
3298
- pending_state_operations: this.stateOperationRequests.size,
3299
- pending_event_waiters: this.liveWaiters.length
3300
- });
3301
- this.emitLifecycleEvent("allocations_finished", "Allocations completed");
3302
- this.finish();
3303
- this.replay?.dispose();
3304
- }
3305
- }
3306
- async finishUserError(error) {
3307
- const rendered = error instanceof Error ? error.message : String(error);
3308
- this.runWithoutReplayOwnership(() => {
3309
- printCloudEvent({
3310
- level: "error",
3311
- event: "function_call_failed",
3312
- message: rendered,
3313
- namespace: this.functionRef.namespace,
3314
- application: this.functionRef.applicationName,
3315
- application_version: this.functionRef.applicationVersion,
3316
- function: this.functionRef.functionName,
3317
- request_id: this.allocation.requestId,
3318
- function_call_id: this.allocation.functionCallId,
3319
- allocation_id: this.allocation.allocationId,
3320
- error: error instanceof Error ? error.stack?.split("\n") ?? [rendered] : [rendered]
3321
- });
3322
- });
3323
- if (isRequestError(error)) {
3324
- const output = prepareTextObject(error.message);
3325
- const requestErrorBlob = this.allocation.inputs?.requestErrorBlob;
3326
- if (requestErrorBlob == null) throw new Error("Allocation has no request error BLOB");
3327
- const uploaded = await this.uploadBlob(requestErrorBlob, output.bytes, "request error output");
3328
- this.log("info", "queueing request-error allocation finish", {
3329
- request_error_bytes: output.bytes.byteLength,
3330
- request_error_blob_id: uploaded.id
3331
- });
3332
- this.queueTerminalBatch({
3333
- outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
3334
- failureReason: "ALLOCATION_FAILURE_REASON_REQUEST_ERROR",
3335
- requestErrorOutput: output.object,
3336
- uploadedRequestErrorBlob: uploaded
3337
- });
3338
- return;
3339
- }
3340
- this.log("info", "queueing function-error allocation finish", {}, error);
3341
- this.queueTerminalBatch({
3342
- outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
3343
- failureReason: "ALLOCATION_FAILURE_REASON_FUNCTION_ERROR"
3344
- });
3345
- }
3346
- createRequestContext() {
3347
- return {
3348
- requestId: String(this.allocation.requestId),
3349
- headers: new Headers2(
3350
- (this.allocation.inputs?.requestContext?.headers ?? []).map(
3351
- (header) => [
3352
- String(header.name ?? ""),
3353
- String(header.value ?? "")
3354
- ]
3355
- )
3356
- ),
3357
- signal: this.controller.signal,
3358
- state: {
3359
- get: async (key, defaultValue) => {
3360
- validateRequestStateKey(key);
3361
- const operationId = (0, import_node_crypto5.randomUUID)();
3362
- this.log("debug", "request state read starting", {
3363
- state_key: key,
3364
- operation_id: operationId,
3365
- has_default: defaultValue !== void 0
3366
- });
3367
- const result = await this.requestStateOperation({ operationId, stateKey: key, prepareRead: {} });
3368
- if (result.status?.code === NOT_FOUND) {
3369
- this.log("debug", "request state key not found", { state_key: key, operation_id: operationId });
3370
- return defaultValue;
3371
- }
3372
- const error = statusError(result.status, `Failed to read request state '${key}'`);
3373
- if (error != null) throw error;
3374
- const data = await this.downloadBlob(result.prepareRead?.blob, "request state value", {
3375
- state_key: key,
3376
- operation_id: operationId
3377
- });
3378
- const value = deserializeJSON(data);
3379
- this.log("debug", "request state read completed", {
3380
- state_key: key,
3381
- operation_id: operationId,
3382
- value_bytes: data.byteLength
3383
- });
3384
- return value;
3385
- },
3386
- set: async (key, value) => {
3387
- validateRequestStateKey(key);
3388
- const serialized = serializeRequestStateValue(value);
3389
- const prepareOperationId = (0, import_node_crypto5.randomUUID)();
3390
- this.log("debug", "request state write preparation starting", {
3391
- state_key: key,
3392
- operation_id: prepareOperationId,
3393
- value_bytes: serialized.byteLength
3394
- });
3395
- const prepared = await this.requestStateOperation({
3396
- operationId: prepareOperationId,
3397
- stateKey: key,
3398
- prepareWrite: { size: serialized.byteLength }
3399
- });
3400
- const error = statusError(prepared.status, `Failed to prepare request state '${key}'`);
3401
- if (error != null) throw error;
3402
- const blob = await this.uploadBlob(prepared.prepareWrite?.blob, serialized, "request state value", {
3403
- state_key: key,
3404
- operation_id: prepareOperationId
3405
- });
3406
- const commitOperationId = (0, import_node_crypto5.randomUUID)();
3407
- this.log("debug", "request state write commit starting", {
3408
- state_key: key,
3409
- operation_id: commitOperationId,
3410
- blob_id: blob.id
3411
- });
3412
- const committed = await this.requestStateOperation({
3413
- operationId: commitOperationId,
3414
- stateKey: key,
3415
- commitWrite: { blob }
3416
- });
3417
- const commitError = statusError(committed.status, `Failed to commit request state '${key}'`);
3418
- if (commitError != null) throw commitError;
3419
- this.log("debug", "request state write completed", {
3420
- state_key: key,
3421
- operation_id: commitOperationId,
3422
- value_bytes: serialized.byteLength
3423
- });
3424
- }
3425
- },
3426
- metrics: {
3427
- counter: async (name, value = 1) => {
3428
- validateCounterMetric(name, value);
3429
- this.emitUserEvent("ai.tensorlake.metric.counter.inc", {
3430
- counter_name: name,
3431
- counter_inc: value
3432
- });
3433
- },
3434
- timer: async (name, value) => {
3435
- validateTimerMetric(name, value);
3436
- this.emitUserEvent("ai.tensorlake.metric.timer", {
3437
- timer_name: name,
3438
- timer_value: value
3439
- });
3440
- }
3441
- },
3442
- progress: {
3443
- update: async (current, total, options) => {
3444
- validateProgressUpdate(current, total, options);
3445
- this.log("debug", "request progress update", {
3446
- current,
3447
- total,
3448
- has_custom_message: options?.message != null,
3449
- attribute_count: Object.keys(options?.attributes ?? {}).length
3450
- });
3451
- this.state.progress = { current, total };
3452
- this.publishState();
3453
- const message = options?.message ?? `${this.functionRef.functionName}: executing step ${current} of ${total}`;
3454
- this.runWithoutReplayOwnership(() => {
3455
- printCloudEvent({ RequestProgressUpdated: {
3456
- request_id: this.allocation.requestId,
3457
- function_name: this.functionRef.functionName,
3458
- function_run_id: this.allocation.functionCallId,
3459
- allocation_id: this.allocation.allocationId,
3460
- message,
3461
- step: current,
3462
- total,
3463
- attributes: options?.attributes,
3464
- created_at: (/* @__PURE__ */ new Date()).toISOString()
3465
- } }, {
3466
- type: "ai.tensorlake.progress_update",
3467
- source: "/tensorlake/applications/progress",
3468
- message
3469
- });
3470
- });
3471
- }
3472
- }
3473
- };
3474
- }
3475
- emitUserEvent(type, data) {
3476
- this.log("debug", "user metric event emitted", {
3477
- event_type: type,
3478
- metric_name: data.counter_name ?? data.timer_name
3479
- });
3480
- this.runWithoutReplayOwnership(() => {
3481
- printCloudEvent({
3482
- request_id: this.allocation.requestId,
3483
- function_name: this.functionRef.functionName,
3484
- function_run_id: this.allocation.functionCallId,
3485
- allocation_id: this.allocation.allocationId,
3486
- ...data
3487
- }, { type, source: "/tensorlake/applications/metrics" });
3488
- });
3489
- }
3490
- emitLifecycleEvent(event, message) {
3491
- this.runWithoutReplayOwnership(() => {
3492
- printCloudEvent({
3493
- event,
3494
- message,
3495
- allocations: [{
3496
- namespace: this.functionRef.namespace,
3497
- application: this.functionRef.applicationName,
3498
- application_version: this.functionRef.applicationVersion,
3499
- function: this.functionRef.functionName,
3500
- request_id: this.allocation.requestId,
3501
- function_call_id: this.allocation.functionCallId,
3502
- allocation_id: this.allocation.allocationId
3503
- }]
3504
- });
3505
- });
3506
- }
3507
- async loadReplayHistory() {
3508
- const entries = [];
3509
- let afterClock = 0;
3510
- let page = 0;
3511
- while (true) {
3512
- page += 1;
3513
- this.log("debug", "requesting replay history page", { page, after_clock: afterClock });
3514
- const requestedAfterClock = afterClock;
3515
- const response = await this.readEvents(afterClock);
3516
- entries.push(...response.entries ?? []);
3517
- this.log("debug", "replay history page received", {
3518
- page,
3519
- entry_count: response.entries?.length ?? 0,
3520
- last_clock: response.lastClock,
3521
- has_more: response.hasMore
3522
- });
3523
- const responseClock = validatedEventLogClock(
3524
- response,
3525
- requestedAfterClock,
3526
- (message) => new ReplayMismatchError(message)
3527
- );
3528
- afterClock = responseClock;
3529
- if (!response.hasMore) break;
3530
- }
3531
- this.lastEventClock = afterClock;
3532
- this.replay = new ReplayHistory(entries);
3533
- this.log("info", "strict replay history loaded", {
3534
- page_count: page,
3535
- entry_count: entries.length,
3536
- last_clock: afterClock
3537
- });
3538
- }
3539
- waitForLiveEvent(predicate) {
3540
- if (this.controller.signal.aborted) {
3541
- return Promise.reject(this.controller.signal.reason);
3542
- }
3543
- const index = this.liveBacklog.findIndex(predicate);
3544
- if (index >= 0) {
3545
- this.log("debug", "durable event satisfied from backlog", {
3546
- backlog_size: this.liveBacklog.length,
3547
- pending_waiters: this.liveWaiters.length
3548
- });
3549
- return Promise.resolve(this.liveBacklog.splice(index, 1)[0]);
3550
- }
3551
- const result = deferred();
3552
- this.liveWaiters.push({ predicate, result });
3553
- this.log("debug", "waiting for durable event", {
3554
- backlog_size: this.liveBacklog.length,
3555
- pending_waiters: this.liveWaiters.length
3556
- });
3557
- this.ensureLivePump();
3558
- return result.promise;
3559
- }
3560
- ensureLivePump() {
3561
- if (this.livePump != null) {
3562
- this.log("debug", "durable event live pump already running");
3563
- return;
3564
- }
3565
- this.log("info", "durable event live pump starting", { after_clock: this.lastEventClock });
3566
- const pump = (async () => {
3567
- while (!this.finished && this.liveWaiters.length > 0) {
3568
- const requestedAfterClock = this.lastEventClock;
3569
- const response = await this.readEvents(this.lastEventClock);
3570
- this.log("debug", "durable event log page received", {
3571
- requested_after_clock: requestedAfterClock,
3572
- entry_count: response.entries?.length ?? 0,
3573
- last_clock: response.lastClock,
3574
- has_more: response.hasMore
3575
- });
3576
- const responseClock = validatedEventLogClock(
3577
- response,
3578
- requestedAfterClock,
3579
- (message) => new AllocationProtocolError(message)
3580
- );
3581
- this.lastEventClock = responseClock;
3582
- for (const entry of response.entries ?? []) this.dispatchLiveEvent(entry);
3583
- }
3584
- })();
3585
- this.livePump = pump;
3586
- void pump.then(() => {
3587
- if (this.livePump !== pump) return;
3588
- this.livePump = void 0;
3589
- this.log("debug", "durable event live pump stopped", {
3590
- finished: this.finished,
3591
- pending_waiters: this.liveWaiters.length,
3592
- after_clock: this.lastEventClock
3593
- });
3594
- if (!this.finished && this.liveWaiters.length > 0) this.ensureLivePump();
3595
- }, (error) => {
3596
- if (this.livePump === pump) this.livePump = void 0;
3597
- if (!this.finished && !this.controller.signal.aborted) {
3598
- const protocolError = error instanceof AllocationProtocolError ? error : new AllocationProtocolError(
3599
- error instanceof Error ? error.message : String(error),
3600
- { cause: error }
3601
- );
3602
- this.latchProtocolError(protocolError);
3603
- this.log("error", "durable event live pump failed", {
3604
- pending_waiters: this.liveWaiters.length
3605
- }, protocolError);
3606
- }
3607
- });
3608
- }
3609
- dispatchLiveEvent(entry) {
3610
- const index = this.liveWaiters.findIndex((waiter) => waiter.predicate(entry));
3611
- const eventKind = Object.keys(entry).find((key) => key !== "clock") ?? "unknown";
3612
- if (index < 0) {
3613
- this.liveBacklog.push(entry);
3614
- this.log("debug", "durable event added to backlog", {
3615
- event_kind: eventKind,
3616
- event_clock: entry.clock,
3617
- backlog_size: this.liveBacklog.length
3618
- });
3619
- } else {
3620
- this.liveWaiters.splice(index, 1)[0].result.resolve(entry);
3621
- this.log("debug", "durable event delivered to waiter", {
3622
- event_kind: eventKind,
3623
- event_clock: entry.clock,
3624
- pending_waiters: this.liveWaiters.length
3625
- });
3626
- }
3627
- }
3628
- readEvents(afterClock) {
3629
- if (this.currentRead != null) {
3630
- throw new AllocationProtocolError("Allocation event log already has a pending read");
3631
- }
3632
- const response = deferred();
3633
- const request = {
3634
- allocationId: this.allocation.allocationId,
3635
- afterClock,
3636
- maxEntries: 100
3637
- };
3638
- this.currentRead = { request, response };
3639
- this.log("debug", "event log read requested", {
3640
- after_clock: afterClock,
3641
- max_entries: request.maxEntries,
3642
- connected_streams: this.eventReadStreams.size
3643
- });
3644
- for (const stream of [...this.eventReadStreams]) {
3645
- try {
3646
- stream.write(request);
3647
- } catch (error) {
3648
- this.dropEventReadStream(stream, "write_failed", error);
3649
- }
3650
- }
3651
- return response.promise;
3652
- }
3653
- watchEventLogReads(stream) {
3654
- this.eventReadStreams.add(stream);
3655
- this.log("info", "event log read stream connected", {
3656
- connected_streams: this.eventReadStreams.size,
3657
- has_pending_read: this.currentRead != null
3658
- });
3659
- try {
3660
- stream.on("cancelled", () => this.dropEventReadStream(stream, "cancelled"));
3661
- stream.on("close", () => this.dropEventReadStream(stream, "closed"));
3662
- if (this.currentRead != null) {
3663
- this.log("debug", "sending pending event log read to new stream", {
3664
- after_clock: this.currentRead.request.afterClock,
3665
- max_entries: this.currentRead.request.maxEntries
3666
- });
3667
- stream.write(this.currentRead.request);
3668
- }
3669
- if (this.finished && this.currentRead == null) stream.end();
3670
- } catch (error) {
3671
- this.dropEventReadStream(stream, "setup_failed", error);
3672
- throw error;
3673
- }
3674
- }
3675
- deliverEventLogResponse(response) {
3676
- if (this.currentRead == null) {
3677
- this.log("warn", "unexpected event log response ignored", {
3678
- entry_count: response.entries?.length ?? 0,
3679
- last_clock: response.lastClock
3680
- });
3681
- return;
3682
- }
3683
- const pending = this.currentRead;
3684
- this.currentRead = void 0;
3685
- this.log("debug", "event log response delivered", {
3686
- requested_after_clock: pending.request.afterClock,
3687
- entry_count: response.entries?.length ?? 0,
3688
- last_clock: response.lastClock,
3689
- has_more: response.hasMore
3690
- });
3691
- pending.response.resolve(response);
3692
- }
3693
- requestOutputBlob(size) {
3694
- if (this.controller.signal.aborted) {
3695
- return Promise.reject(this.controller.signal.reason);
3696
- }
3697
- const id = (0, import_node_crypto5.randomUUID)();
3698
- const request = deferred();
3699
- this.outputBlobRequests.set(id, request);
3700
- this.state.outputBlobRequests.push({ id, size });
3701
- this.log("debug", "output blob requested", {
3702
- blob_id: id,
3703
- requested_bytes: size,
3704
- pending_output_blob_requests: this.outputBlobRequests.size
3705
- });
3706
- this.publishState();
3707
- return request.promise.then((output) => {
3708
- this.log("debug", "output blob response received", {
3709
- blob_id: id,
3710
- status_code: output.status?.code,
3711
- chunk_count: output.blob?.chunks?.length ?? 0
3712
- });
3713
- const error = statusError(output.status, "Failed to create output BLOB");
3714
- if (error != null) throw error;
3715
- return output.blob;
3716
- });
3717
- }
3718
- requestStateOperation(operation) {
3719
- if (this.controller.signal.aborted) {
3720
- return Promise.reject(this.controller.signal.reason);
3721
- }
3722
- const request = deferred();
3723
- this.stateOperationRequests.set(operation.operationId, request);
3724
- this.state.requestStateOperations.push(operation);
3725
- this.log("debug", "request state operation queued", {
3726
- operation_id: operation.operationId,
3727
- state_key: operation.stateKey,
3728
- operation_kind: this.stateOperationKind(operation),
3729
- requested_bytes: operation.prepareWrite?.size,
3730
- pending_state_operations: this.stateOperationRequests.size
3731
- });
3732
- this.publishState();
3733
- return request.promise;
3734
- }
3735
- deliverUpdate(update) {
3736
- if (update.outputBlob != null) {
3737
- let id = String(update.outputBlob.blob?.id ?? "");
3738
- let matchedByOrder = false;
3739
- if (!id) {
3740
- id = String(this.state.outputBlobRequests[0]?.id ?? "");
3741
- matchedByOrder = id !== "";
3742
- }
3743
- const request = this.outputBlobRequests.get(id);
3744
- if (request == null) {
3745
- this.log("warn", "unmatched output blob update ignored", {
3746
- blob_id: id || void 0,
3747
- status_code: update.outputBlob.status?.code,
3748
- pending_output_blob_requests: this.outputBlobRequests.size
3749
- });
3750
- return;
3751
- }
3752
- this.outputBlobRequests.delete(id);
3753
- this.state.outputBlobRequests = this.state.outputBlobRequests.filter((item) => item.id !== id);
3754
- this.log("debug", "output blob update matched", {
3755
- blob_id: id,
3756
- status_code: update.outputBlob.status?.code,
3757
- chunk_count: update.outputBlob.blob?.chunks?.length ?? 0,
3758
- matched_by_order: matchedByOrder,
3759
- pending_output_blob_requests: this.outputBlobRequests.size
3760
- });
3761
- request.resolve(update.outputBlob);
3762
- this.publishState();
3763
- return;
3764
- }
3765
- const result = update.requestStateOperationResult;
3766
- if (result != null) {
3767
- if (typeof result.operationId !== "string" || result.operationId.length === 0) {
3768
- throw this.latchProtocolError(
3769
- "Allocation received a request state operation result without an operation ID"
3770
- );
3771
- }
3772
- const id = result.operationId;
3773
- const request = this.stateOperationRequests.get(id);
3774
- if (request == null) {
3775
- this.log("warn", "unmatched request state operation result ignored", {
3776
- operation_id: id,
3777
- status_code: result.status?.code,
3778
- pending_state_operations: this.stateOperationRequests.size
3779
- });
3780
- return;
3781
- }
3782
- const operation = this.state.requestStateOperations.find((item) => item.operationId === id);
3783
- this.stateOperationRequests.delete(id);
3784
- this.state.requestStateOperations = this.state.requestStateOperations.filter(
3785
- (item) => item.operationId !== id
3786
- );
3787
- this.log("debug", "request state operation result matched", {
3788
- operation_id: id,
3789
- state_key: operation?.stateKey,
3790
- operation_kind: this.stateOperationKind(operation),
3791
- status_code: result.status?.code,
3792
- pending_state_operations: this.stateOperationRequests.size
3793
- });
3794
- request.resolve(result);
3795
- this.publishState();
3796
- return;
3797
- }
3798
- this.log("warn", "allocation update contained no recognized payload");
3799
- }
3800
- watchState(stream) {
3801
- this.stateStreams.add(stream);
3802
- this.stateStreamFlows.set(stream, { backpressured: false });
3803
- this.log("info", "allocation state stream connected", {
3804
- connected_streams: this.stateStreams.size,
3805
- state_hash: this.state.sha256Hash,
3806
- pending_output_blob_requests: this.outputBlobRequests.size,
3807
- pending_state_operations: this.stateOperationRequests.size
3808
- });
3809
- try {
3810
- stream.on("cancelled", () => this.dropStateStream(stream, "cancelled"));
3811
- stream.on("close", () => this.dropStateStream(stream, "closed"));
3812
- stream.on("drain", () => this.drainStateStream(stream));
3813
- this.writeStateSnapshot(stream, this.snapshotState());
3814
- if (this.finished) stream.end();
3815
- } catch (error) {
3816
- this.dropStateStream(stream, "setup_failed", error);
3817
- throw error;
3818
- }
3819
- }
3820
- updateStateHash() {
3821
- const content = JSON.stringify({
3822
- progress: this.state.progress,
3823
- outputBlobRequests: this.state.outputBlobRequests,
3824
- requestStateOperations: this.state.requestStateOperations
3825
- });
3826
- this.state.sha256Hash = (0, import_node_crypto5.createHash)("sha256").update(content).digest("hex");
3827
- }
3828
- snapshotState() {
3829
- return nativeStructuredClone(this.state);
3830
- }
3831
- publishState() {
3832
- this.updateStateHash();
3833
- this.log("debug", "allocation state published", {
3834
- state_hash: this.state.sha256Hash,
3835
- connected_streams: this.stateStreams.size,
3836
- pending_output_blob_requests: this.outputBlobRequests.size,
3837
- pending_state_operations: this.stateOperationRequests.size,
3838
- progress_current: this.state.progress?.current,
3839
- progress_total: this.state.progress?.total
3840
- });
3841
- for (const stream of [...this.stateStreams]) {
3842
- this.writeStateSnapshot(stream, this.snapshotState());
3843
- }
3844
- }
3845
- writeStateSnapshot(stream, snapshot) {
3846
- const flow = this.stateStreamFlows.get(stream);
3847
- if (flow == null) return;
3848
- if (flow.backpressured) {
3849
- flow.pending = snapshot;
3850
- return;
3851
- }
3852
- try {
3853
- flow.backpressured = !stream.write(snapshot);
3854
- } catch (error) {
3855
- this.dropStateStream(stream, "write_failed", error);
3856
- }
3857
- }
3858
- drainStateStream(stream) {
3859
- const flow = this.stateStreamFlows.get(stream);
3860
- if (flow == null) return;
3861
- flow.backpressured = false;
3862
- const pending = flow.pending;
3863
- flow.pending = void 0;
3864
- if (pending != null) this.writeStateSnapshot(stream, pending);
3865
- }
3866
- addBatch(events) {
3867
- if (this.terminalBatchQueued && events.every((event) => event.finishAllocation == null)) {
3868
- this.log("error", "execution event rejected after terminal batch", {
3869
- event_count: events.length,
3870
- event_kinds: events.map((event) => Object.keys(event)[0] ?? "unknown")
3871
- });
3872
- return;
3873
- }
3874
- this.executionBatches.push(events);
3875
- this.log("debug", "execution log batch queued", {
3876
- event_count: events.length,
3877
- event_kinds: events.map((event) => Object.keys(event)[0] ?? "unknown"),
3878
- queued_batches: this.executionBatches.length,
3879
- waiting_consumers: this.executionBatchWaiters.length
3880
- });
3881
- for (const waiter of this.executionBatchWaiters.splice(0)) {
3882
- this.markTerminalBatchObserved(events);
3883
- waiter(events);
3884
- }
3885
- }
3886
- queueTerminalBatch(finishAllocation) {
3887
- if (this.terminalBatchQueued) {
3888
- this.log("error", "duplicate terminal allocation batch rejected", {
3889
- outcome_code: finishAllocation.outcomeCode,
3890
- failure_reason: finishAllocation.failureReason
3891
- });
3892
- return;
3893
- }
3894
- this.terminalBatchQueued = true;
3895
- this.addBatch([{ finishAllocation }]);
3896
- }
3897
- async getExecutionBatch() {
3898
- if (this.executionBatches.length > 0) {
3899
- this.log("debug", "returning queued execution log batch", {
3900
- event_count: this.executionBatches[0].length,
3901
- queued_batches: this.executionBatches.length
3902
- });
3903
- const events = this.executionBatches[0];
3904
- this.markTerminalBatchObserved(events);
3905
- return events;
3906
- }
3907
- if (this.finished) {
3908
- this.log("debug", "returning empty execution log batch for finished allocation");
3909
- return [];
3910
- }
3911
- this.log("debug", "execution log batch consumer waiting", {
3912
- waiting_consumers: this.executionBatchWaiters.length + 1
3913
- });
3914
- return new Promise((resolve) => this.executionBatchWaiters.push(resolve));
3915
- }
3916
- advanceExecutionBatch() {
3917
- const advanced = this.executionBatches.shift();
3918
- this.log("debug", "execution log batch advanced", {
3919
- advanced_event_count: advanced?.length ?? 0,
3920
- queued_batches: this.executionBatches.length
3921
- });
3922
- }
3923
- markTerminalBatchObserved(events) {
3924
- if (this.terminalBatchObserved || !events.some((event) => event.finishAllocation != null)) {
3925
- return;
3926
- }
3927
- this.terminalBatchObserved = true;
3928
- this.log("debug", "terminal execution log batch observed by consumer");
3929
- for (const waiter of this.terminalBatchObservedWaiters.splice(0)) waiter();
3930
- }
3931
- get isFinished() {
3932
- return this.finished;
3933
- }
3934
- finish() {
3935
- if (this.finished) {
3936
- this.log("debug", "allocation finish ignored because runner is already finished");
3937
- return;
3938
- }
3939
- this.finished = true;
3940
- this.abortExecution(new FunctionError("Allocation finished"));
3941
- this.log("info", "allocation runner finished", {
3942
- duration_ms: Date.now() - this.startedAt,
3943
- queued_execution_batches: this.executionBatches.length,
3944
- state_streams: this.stateStreams.size,
3945
- event_log_streams: this.eventReadStreams.size
3946
- });
3947
- for (const stream of [...this.stateStreams]) {
3948
- try {
3949
- stream.end();
3950
- } catch (error) {
3951
- this.dropStateStream(stream, "end_failed", error);
3952
- }
3953
- }
3954
- for (const stream of [...this.eventReadStreams]) {
3955
- try {
3956
- stream.end();
3957
- } catch (error) {
3958
- this.dropEventReadStream(stream, "end_failed", error);
3959
- }
3960
- }
3961
- for (const waiter of this.executionBatchWaiters.splice(0)) waiter([]);
3962
- }
3963
- dropStateStream(stream, reason, error) {
3964
- if (!this.stateStreams.delete(stream)) return;
3965
- this.stateStreamFlows.delete(stream);
3966
- this.log("info", "allocation state stream disconnected", {
3967
- reason,
3968
- connected_streams: this.stateStreams.size
3969
- }, error);
3970
- }
3971
- dropEventReadStream(stream, reason, error) {
3972
- if (!this.eventReadStreams.delete(stream)) return;
3973
- this.log("info", "event log read stream disconnected", {
3974
- reason,
3975
- connected_streams: this.eventReadStreams.size,
3976
- has_pending_read: this.currentRead != null
3977
- }, error);
3978
- }
3979
- stateOperationKind(operation) {
3980
- if (operation?.prepareRead != null) return "prepare_read";
3981
- if (operation?.prepareWrite != null) return "prepare_write";
3982
- if (operation?.commitWrite != null) return "commit_write";
3983
- return "unknown";
3984
- }
3985
- blobLogFields(blob) {
3986
- const chunks = blob?.chunks ?? [];
3987
- return {
3988
- blob_id: blob?.id,
3989
- chunk_count: chunks.length,
3990
- declared_bytes: chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0)
3991
- };
3992
- }
3993
- async downloadBlob(blob, purpose, fields = {}) {
3994
- if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
3995
- const startedAt = Date.now();
3996
- this.log("debug", "blob download starting", {
3997
- purpose,
3998
- ...fields,
3999
- ...this.blobLogFields(blob)
4000
- });
4001
- try {
4002
- const data = await downloadBlob(
4003
- blob,
4004
- (message, details) => this.log("debug", `blob ${message}`, {
4005
- purpose,
4006
- ...fields,
4007
- ...details
4008
- }),
4009
- this.controller.signal
4010
- );
4011
- this.log("debug", "blob download completed", {
4012
- purpose,
4013
- ...fields,
4014
- ...this.blobLogFields(blob),
4015
- downloaded_bytes: data.byteLength,
4016
- duration_ms: Date.now() - startedAt
4017
- });
4018
- return data;
4019
- } catch (error) {
4020
- this.log("error", "blob download failed", {
4021
- purpose,
4022
- ...fields,
4023
- ...this.blobLogFields(blob),
4024
- duration_ms: Date.now() - startedAt
4025
- }, error);
4026
- throw error;
4027
- }
4028
- }
4029
- async downloadSerializedObject(object, blob, purpose, fields = {}) {
4030
- if (object == null) throw new Error(`Missing serialized object for ${purpose}`);
4031
- if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
4032
- const startedAt = Date.now();
4033
- this.log("debug", "serialized object download starting", {
4034
- purpose,
4035
- ...fields,
4036
- ...this.blobLogFields(blob),
4037
- object_offset: object.offset ?? 0,
4038
- object_bytes: object.manifest?.size,
4039
- metadata_bytes: object.manifest?.metadataSize,
4040
- content_type: object.manifest?.contentType,
4041
- encoding: object.manifest?.encoding
4042
- });
4043
- try {
4044
- const value = await downloadSerializedObject(
4045
- object,
4046
- blob,
4047
- (message, details) => this.log(
4048
- "debug",
4049
- `blob ${message}`,
4050
- { purpose, ...fields, ...details }
4051
- ),
4052
- this.controller.signal
4053
- );
4054
- this.log("debug", "serialized object download completed", {
4055
- purpose,
4056
- ...fields,
4057
- ...this.blobLogFields(blob),
4058
- object_bytes: value.data.byteLength + value.metadata.byteLength,
4059
- data_bytes: value.data.byteLength,
4060
- metadata_bytes: value.metadata.byteLength,
4061
- duration_ms: Date.now() - startedAt
4062
- });
4063
- return value;
4064
- } catch (error) {
4065
- this.log("error", "serialized object download failed", {
4066
- purpose,
4067
- ...fields,
4068
- ...this.blobLogFields(blob),
4069
- duration_ms: Date.now() - startedAt
4070
- }, error);
4071
- throw error;
4072
- }
4073
- }
4074
- async uploadBlob(blob, data, purpose, fields = {}) {
4075
- if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
4076
- const startedAt = Date.now();
4077
- this.log("debug", "blob upload starting", {
4078
- purpose,
4079
- ...fields,
4080
- ...this.blobLogFields(blob),
4081
- upload_bytes: data.byteLength
4082
- });
4083
- try {
4084
- const uploaded = await uploadBlob(
4085
- blob,
4086
- data,
4087
- (message, details) => this.log("debug", `blob ${message}`, {
4088
- purpose,
4089
- ...fields,
4090
- ...details
4091
- }),
4092
- this.controller.signal
4093
- );
4094
- this.log("debug", "blob upload completed", {
4095
- purpose,
4096
- ...fields,
4097
- ...this.blobLogFields(uploaded),
4098
- upload_bytes: data.byteLength,
4099
- duration_ms: Date.now() - startedAt
4100
- });
4101
- return uploaded;
4102
- } catch (error) {
4103
- this.log("error", "blob upload failed", {
4104
- purpose,
4105
- ...fields,
4106
- ...this.blobLogFields(blob),
4107
- upload_bytes: data.byteLength,
4108
- duration_ms: Date.now() - startedAt
4109
- }, error);
4110
- throw error;
4111
- }
4112
- }
4113
- log(level, message, fields = {}, error) {
4114
- this.runWithoutReplayOwnership(() => {
4115
- writeStructuredOutput("stderr", () => {
4116
- const rendered = error instanceof Error ? error.stack?.split("\n") ?? [`${error.name}: ${error.message}`] : error == null ? void 0 : [String(error)];
4117
- return {
4118
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4119
- level,
4120
- component: "typescript_function_executor_allocation",
4121
- message,
4122
- allocation_id: this.allocation.allocationId,
4123
- request_id: this.allocation.requestId,
4124
- function_call_id: this.allocation.functionCallId,
4125
- namespace: this.functionRef.namespace,
4126
- application: this.functionRef.applicationName,
4127
- application_version: this.functionRef.applicationVersion,
4128
- function: this.functionRef.functionName,
4129
- elapsed_ms: Date.now() - this.startedAt,
4130
- ...fields,
4131
- ...rendered == null ? {} : { error: rendered }
4132
- };
4133
- });
4134
- });
4135
- }
4136
- runWithoutReplayOwnership(callback) {
4137
- return this.replay?.runWithoutCausalOwnership(callback) ?? callback();
4138
- }
4139
- };
4140
-
4141
- // src/function-executor/service.ts
4142
- var SHUTDOWN_TERMINAL_DRAIN_TIMEOUT_MS = 1e3;
4143
- var nativeImport = (0, import_node_vm.compileFunction)(
4144
- "return import(specifier)",
4145
- ["specifier"],
4146
- { importModuleDynamically: import_node_vm.constants.USE_MAIN_CONTEXT_DEFAULT_LOADER }
4147
- );
4148
- function rpcError(code, details) {
4149
- return Object.assign(new Error(details), { code, details, metadata: new grpc.Metadata() });
4150
- }
4151
- function requireSafeSize(value, field) {
4152
- if (!Number.isSafeInteger(value) || Number(value) < 0) {
4153
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field} must be a non-negative safe integer`);
4154
- }
4155
- return Number(value);
4156
- }
4157
- function validateSerializedObjectManifest(manifest, field) {
4158
- if (manifest == null) {
4159
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest is required`);
4160
- }
4161
- if (manifest.encoding == null) {
4162
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.encoding is required`);
4163
- }
4164
- if (manifest.encodingVersion == null) {
4165
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.encoding_version is required`);
4166
- }
4167
- requireSafeSize(manifest.encodingVersion, `${field}.manifest.encoding_version`);
4168
- requireSafeSize(manifest.size, `${field}.manifest.size`);
4169
- if (manifest.metadataSize == null) {
4170
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.metadata_size is required`);
4171
- }
4172
- const metadataSize = requireSafeSize(manifest.metadataSize, `${field}.manifest.metadata_size`);
4173
- if (metadataSize > Number(manifest.size)) {
4174
- throw rpcError(
4175
- grpc.status.INVALID_ARGUMENT,
4176
- `${field}.manifest.metadata_size cannot exceed size`
4177
- );
4178
- }
4179
- if (typeof manifest.sha256Hash !== "string" || manifest.sha256Hash.length === 0) {
4180
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.sha256_hash is required`);
4181
- }
4182
- }
4183
- function validateBlob(blob, field) {
4184
- if (blob == null) throw rpcError(grpc.status.INVALID_ARGUMENT, `${field} is required`);
4185
- if (!Array.isArray(blob.chunks)) {
4186
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.chunks must be an array`);
4187
- }
4188
- blob.chunks.forEach((chunk, index) => {
4189
- if (chunk == null || typeof chunk.uri !== "string" || chunk.uri.length === 0) {
4190
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.chunks[${index}].uri is required`);
4191
- }
4192
- requireSafeSize(chunk.size, `${field}.chunks[${index}].size`);
4193
- });
4194
- }
4195
- function validateNewAllocation(allocation) {
4196
- if (allocation == null) {
4197
- throw rpcError(grpc.status.INVALID_ARGUMENT, "CreateAllocationRequest.allocation is required");
4198
- }
4199
- for (const [field, value] of [
4200
- ["request_id", allocation.requestId],
4201
- ["function_call_id", allocation.functionCallId],
4202
- ["allocation_id", allocation.allocationId]
4203
- ]) {
4204
- if (typeof value !== "string" || value.length === 0) {
4205
- throw rpcError(grpc.status.INVALID_ARGUMENT, `Allocation.${field} is required`);
4206
- }
4207
- }
4208
- const inputs = allocation.inputs;
4209
- if (inputs == null) {
4210
- throw rpcError(grpc.status.INVALID_ARGUMENT, "Allocation.inputs is required");
4211
- }
4212
- if (!Array.isArray(inputs.args) || !Array.isArray(inputs.argBlobs)) {
4213
- throw rpcError(grpc.status.INVALID_ARGUMENT, "Allocation input args and arg_blobs must be arrays");
4214
- }
4215
- if (inputs.args.length !== inputs.argBlobs.length) {
4216
- throw rpcError(
4217
- grpc.status.INVALID_ARGUMENT,
4218
- `Mismatched function arguments and argument blobs lengths, ${inputs.args.length} != ${inputs.argBlobs.length}`
4219
- );
4220
- }
4221
- inputs.args.forEach((argument, index) => {
4222
- if (argument == null) {
4223
- throw rpcError(grpc.status.INVALID_ARGUMENT, `Allocation.inputs.args[${index}] is required`);
4224
- }
4225
- requireSafeSize(argument.offset, `Allocation.inputs.args[${index}].offset`);
4226
- validateSerializedObjectManifest(argument.manifest, `Allocation.inputs.args[${index}]`);
4227
- validateBlob(inputs.argBlobs[index], `Allocation.inputs.arg_blobs[${index}]`);
4228
- });
4229
- validateBlob(inputs.requestErrorBlob, "Allocation.inputs.request_error_blob");
4230
- }
4231
- function safeArchivePath(root, archivePath) {
4232
- const output = import_node_path.default.resolve(root, archivePath);
4233
- if (output !== root && !output.startsWith(`${root}${import_node_path.default.sep}`)) {
4234
- throw new Error(`Application archive contains unsafe path '${archivePath}'`);
4235
- }
4236
- return output;
4237
- }
4238
- async function unpackApplicationCode(data) {
4239
- const root = await (0, import_promises2.mkdtemp)(import_node_path.default.join(import_node_os.default.tmpdir(), "tensorlake-typescript-app-"));
4240
- try {
4241
- const files = (0, import_fflate.unzipSync)(data);
4242
- for (const [name, content] of Object.entries(files)) {
4243
- const destination = safeArchivePath(root, name);
4244
- await (0, import_promises2.mkdir)(import_node_path.default.dirname(destination), { recursive: true });
4245
- await (0, import_promises2.writeFile)(destination, content, { mode: 384 });
4246
- }
4247
- const manifest = JSON.parse(
4248
- await (0, import_promises2.readFile)(import_node_path.default.join(root, ".tensorlake_code_manifest.json"), "utf8")
4249
- );
4250
- return { root, manifest };
4251
- } catch (error) {
4252
- await (0, import_promises2.rm)(root, { recursive: true, force: true });
4253
- throw error;
4254
- }
4255
- }
4256
- var FunctionExecutorService = class {
4257
- allocations = /* @__PURE__ */ new Map();
4258
- functionRef;
4259
- definition;
4260
- applicationDefinition;
4261
- initialization;
4262
- stopping = false;
4263
- shutdownPromise;
4264
- implementation = {
4265
- initialize: this.initialize.bind(this),
4266
- listAllocations: this.listAllocations.bind(this),
4267
- createAllocation: this.createAllocation.bind(this),
4268
- deleteAllocation: this.deleteAllocation.bind(this),
4269
- watchAllocationState: this.watchAllocationState.bind(this),
4270
- sendAllocationUpdate: this.sendAllocationUpdate.bind(this),
4271
- getAllocationExecutionLogBatch: this.getAllocationExecutionLogBatch.bind(this),
4272
- advanceAllocationExecutionLogBatch: this.advanceAllocationExecutionLogBatch.bind(this),
4273
- watchAllocationEventLogReads: this.watchAllocationEventLogReads.bind(this),
4274
- sendAllocationEventLogReadResponse: this.sendAllocationEventLogReadResponse.bind(this),
4275
- checkHealth: this.checkHealth.bind(this),
4276
- getInfo: this.getInfo.bind(this)
4277
- };
4278
- shutdown(reason = new Error("Function executor is shutting down")) {
4279
- if (this.shutdownPromise != null) return this.shutdownPromise;
4280
- this.stopping = true;
4281
- const shutdown = (async () => {
4282
- this.log("info", "function executor service shutdown requested", {
4283
- allocation_count: this.allocations.size
4284
- });
4285
- const allocations = [...this.allocations.values()];
4286
- for (const allocation of allocations) allocation.cancel(reason);
4287
- await Promise.all(allocations.map((allocation) => allocation.waitForCompletion()));
4288
- const terminalObservations = await Promise.all(
4289
- allocations.map(
4290
- (allocation) => allocation.waitForTerminalBatchObserved(SHUTDOWN_TERMINAL_DRAIN_TIMEOUT_MS)
4291
- )
4292
- );
4293
- const unobservedCount = terminalObservations.filter((observed) => !observed).length;
4294
- if (unobservedCount > 0) {
4295
- this.log(
4296
- "warn",
4297
- "shutdown terminal execution batches were not observed before drain deadline",
4298
- { unobserved_allocation_count: unobservedCount }
4299
- );
4300
- }
4301
- this.log("info", "function executor service shutdown completed", {
4302
- allocation_count: allocations.length
4303
- });
4304
- })();
4305
- this.shutdownPromise = shutdown;
4306
- return shutdown;
4307
- }
4308
- initialize(call, callback) {
4309
- if (this.stopping) {
4310
- callback(rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down"));
4311
- return;
4312
- }
4313
- const startedAt = Date.now();
4314
- this.log("info", "initialize RPC received", {
4315
- ...this.functionLogFields(call.request.function),
4316
- application_code_bytes: call.request.applicationCode?.data?.length
4317
- });
4318
- this.emitInitializationEvent(call.request.function, "function_executor_initialization_started", "Initializing function executor");
4319
- void this.initializeAsync(call.request).then(
4320
- () => {
4321
- this.log("info", "initialize RPC completed", {
4322
- ...this.functionLogFields(call.request.function),
4323
- duration_ms: Date.now() - startedAt,
4324
- outcome: "success"
4325
- });
4326
- this.emitInitializationEvent(call.request.function, "function_executor_initialization_finished", "Function executor initialization completed");
4327
- callback(null, { outcomeCode: "INITIALIZATION_OUTCOME_CODE_SUCCESS" });
4328
- },
4329
- (error) => {
4330
- const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
4331
- this.log("error", "initialize RPC failed", {
4332
- ...this.functionLogFields(call.request.function),
4333
- duration_ms: Date.now() - startedAt,
4334
- outcome: "failure"
4335
- }, error);
4336
- this.emitInitializationEvent(
4337
- call.request.function,
4338
- "function_executor_initialization_failed",
4339
- "Function executor initialization failed",
4340
- error
4341
- );
4342
- callback(null, {
4343
- outcomeCode: "INITIALIZATION_OUTCOME_CODE_FAILURE",
4344
- failureReason: "INITIALIZATION_FAILURE_REASON_FUNCTION_ERROR",
4345
- errorMessage: message
4346
- });
4347
- }
4348
- );
4349
- }
4350
- emitInitializationEvent(functionRef, event, message, error) {
4351
- const rendered = error instanceof Error ? error.stack?.split("\n") : error == null ? void 0 : [String(error)];
4352
- printCloudEvent({
4353
- ...error == null ? {} : { level: "error" },
4354
- event,
4355
- message,
4356
- namespace: functionRef?.namespace,
4357
- application: functionRef?.applicationName,
4358
- application_version: functionRef?.applicationVersion,
4359
- function: functionRef?.functionName,
4360
- ...rendered == null ? {} : { error: rendered }
4361
- });
4362
- }
4363
- async initializeAsync(request) {
4364
- if (this.definition != null) throw new Error("Function Executor is already initialized");
4365
- if (this.initialization != null) throw new Error("Function Executor initialization is already in progress");
4366
- const initialization = this.initializeAttempt(request);
4367
- this.initialization = initialization;
4368
- try {
4369
- await initialization;
4370
- } finally {
4371
- if (this.initialization === initialization) this.initialization = void 0;
4372
- }
4373
- }
4374
- async initializeAttempt(request) {
4375
- if (request.function == null) throw new Error("InitializeRequest.function is required");
4376
- for (const [field, value] of [
4377
- ["namespace", request.function.namespace],
4378
- ["application_name", request.function.applicationName],
4379
- ["function_name", request.function.functionName],
4380
- ["application_version", request.function.applicationVersion]
4381
- ]) {
4382
- if (typeof value !== "string" || value.length === 0) {
4383
- throw new Error(`InitializeRequest.function.${field} is required`);
4384
- }
4385
- }
4386
- if (request.applicationCode == null) throw new Error("InitializeRequest.application_code is required");
4387
- if (request.applicationCode.manifest == null) throw new Error("InitializeRequest.application_code.manifest is required");
4388
- if (request.applicationCode.data == null) throw new Error("InitializeRequest.application_code.data is required");
4389
- const codeManifest = request.applicationCode.manifest;
4390
- if (codeManifest.encoding == null) throw new Error("InitializeRequest.application_code.manifest.encoding is required");
4391
- if (codeManifest.encodingVersion == null) throw new Error("InitializeRequest.application_code.manifest.encoding_version is required");
4392
- if (codeManifest.size == null) throw new Error("InitializeRequest.application_code.manifest.size is required");
4393
- if (codeManifest.sha256Hash == null) throw new Error("InitializeRequest.application_code.manifest.sha256_hash is required");
4394
- if (!(/* @__PURE__ */ new Set(["SERIALIZED_OBJECT_ENCODING_BINARY_ZIP", 4])).has(codeManifest.encoding)) {
4395
- throw new Error(
4396
- `Invalid application code encoding: ${String(codeManifest.encoding)}. Expected: BINARY_ZIP`
4397
- );
4398
- }
4399
- const declaredSize = requireSafeSize(
4400
- codeManifest.size,
4401
- "InitializeRequest.application_code.manifest.size"
4402
- );
4403
- const metadataSize = codeManifest.metadataSize == null ? 0 : requireSafeSize(
4404
- codeManifest.metadataSize,
4405
- "InitializeRequest.application_code.manifest.metadata_size"
4406
- );
4407
- if (metadataSize > declaredSize) {
4408
- throw new Error(
4409
- "InitializeRequest.application_code.manifest.metadata_size cannot exceed size"
4410
- );
4411
- }
4412
- const archive = new Uint8Array(request.applicationCode.data);
4413
- this.log("debug", "unpacking application code", {
4414
- ...this.functionLogFields(request.function),
4415
- application_code_bytes: archive.byteLength
4416
- });
4417
- const unpackStartedAt = Date.now();
4418
- const { root, manifest } = await unpackApplicationCode(archive);
4419
- let registrySnapshot;
4420
- try {
4421
- this.log("debug", "application code unpacked", {
4422
- ...this.functionLogFields(request.function),
4423
- duration_ms: Date.now() - unpackStartedAt,
4424
- manifest_format_version: manifest.format_version,
4425
- manifest_runtime: manifest.runtime,
4426
- manifest_function_count: Object.keys(manifest.functions ?? {}).length
4427
- });
4428
- if (manifest.format_version !== 2 || manifest.runtime !== "typescript") {
4429
- throw new Error("Application code is not a Tensorlake TypeScript bundle");
4430
- }
4431
- const nodeMajor = Number(process.versions.node.split(".")[0]);
4432
- if ((manifest.minimum_node_major ?? 24) > nodeMajor) {
4433
- throw new Error(`Application requires Node ${manifest.minimum_node_major} or newer; executor has ${nodeMajor}`);
4434
- }
4435
- const moduleName = manifest.module;
4436
- if (!moduleName) throw new Error("Application code manifest does not name a runtime module");
4437
- const functionName = String(request.function.functionName);
4438
- if (manifest.functions?.[functionName] == null) {
4439
- throw new Error(`Function '${functionName}' is not present in the application bundle`);
4440
- }
4441
- const modulePath = safeArchivePath(root, moduleName);
4442
- this.log("debug", "importing application runtime module", {
4443
- ...this.functionLogFields(request.function),
4444
- module: moduleName
4445
- });
4446
- const importStartedAt = Date.now();
4447
- registrySnapshot = snapshotRegistry();
4448
- const runtime = await nativeImport(`${(0, import_node_url2.pathToFileURL)(modulePath).href}?executor=${Date.now()}`);
4449
- if (typeof runtime.__tensorlakeGetFunction !== "function") {
4450
- throw new Error("Application runtime does not export __tensorlakeGetFunction");
4451
- }
4452
- const definition = runtime.__tensorlakeGetFunction(functionName);
4453
- if (definition?.name !== functionName) {
4454
- throw new Error(
4455
- `Application runtime resolved function '${functionName}' as '${String(definition?.name)}'`
4456
- );
4457
- }
4458
- const applicationName = String(request.function.applicationName);
4459
- const applicationDefinition = runtime.__tensorlakeGetFunction(applicationName);
4460
- if (applicationDefinition?.name !== applicationName) {
4461
- throw new Error(
4462
- `Application runtime resolved application '${applicationName}' as '${String(applicationDefinition?.name)}'`
4463
- );
4464
- }
4465
- if (applicationDefinition.application == null) {
4466
- throw new Error(`'${applicationName}' is not a Tensorlake application`);
4467
- }
4468
- this.functionRef = request.function;
4469
- this.definition = definition;
4470
- this.applicationDefinition = applicationDefinition;
4471
- this.log("debug", "application function resolved", {
4472
- ...this.functionLogFields(request.function),
4473
- duration_ms: Date.now() - importStartedAt,
4474
- parameter_count: definition.parameters.length,
4475
- application_max_retries: applicationDefinition.application.retries.maxRetries
4476
- });
4477
- registrySnapshot = void 0;
4478
- } catch (error) {
4479
- if (registrySnapshot != null) restoreRegistry(registrySnapshot);
4480
- this.functionRef = void 0;
4481
- this.definition = void 0;
4482
- this.applicationDefinition = void 0;
4483
- await (0, import_promises2.rm)(root, { recursive: true, force: true });
4484
- throw error;
4485
- }
4486
- }
4487
- getAllocation(id) {
4488
- const allocation = this.allocations.get(id);
4489
- if (allocation == null) throw rpcError(grpc.status.NOT_FOUND, `Allocation ${id} not found`);
4490
- return allocation;
4491
- }
4492
- listAllocations(_call, callback) {
4493
- this.log("debug", "list allocations RPC received", { allocation_count: this.allocations.size });
4494
- callback(null, { allocations: [...this.allocations.values()].map((runner) => runner.allocation) });
4495
- }
4496
- createAllocation(call, callback) {
4497
- void this.createAllocationAsync(call).then(
4498
- () => callback(null, {}),
4499
- (error) => {
4500
- this.log(
4501
- "error",
4502
- "create allocation RPC failed",
4503
- this.allocationLogFields(call.request.allocation),
4504
- error
4505
- );
4506
- callback(error);
4507
- }
4508
- );
4509
- }
4510
- async createAllocationAsync(call) {
4511
- if (this.stopping) {
4512
- throw rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down");
4513
- }
4514
- const initialization = this.initialization;
4515
- if (initialization != null) {
4516
- this.log("info", "create allocation RPC waiting for initialization", {
4517
- ...this.allocationLogFields(call.request.allocation)
4518
- });
4519
- await this.waitForInitialization(call, initialization);
4520
- this.log("info", "create allocation RPC resumed after initialization", {
4521
- ...this.allocationLogFields(call.request.allocation)
4522
- });
4523
- }
4524
- this.assertAllocationAdmissionActive(call);
4525
- if (this.stopping) {
4526
- throw rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down");
4527
- }
4528
- if (this.definition == null || this.applicationDefinition == null || this.functionRef == null) {
4529
- throw rpcError(grpc.status.FAILED_PRECONDITION, "Function Executor is not initialized");
4530
- }
4531
- const allocation = call.request.allocation;
4532
- validateNewAllocation(allocation);
4533
- const id = String(allocation.allocationId);
4534
- this.log("info", "create allocation RPC received", {
4535
- ...this.allocationLogFields(allocation),
4536
- replay_mode: allocation?.replayMode,
4537
- input_count: allocation?.inputs?.args?.length ?? 0
4538
- });
4539
- if (this.allocations.has(id)) throw rpcError(grpc.status.ALREADY_EXISTS, `Allocation ${id} already exists`);
4540
- const runner = new AllocationRunner(
4541
- allocation,
4542
- this.functionRef,
4543
- this.definition,
4544
- this.applicationDefinition
4545
- );
4546
- this.allocations.set(id, runner);
4547
- runner.start();
4548
- this.log("info", "allocation runner started", this.allocationLogFields(allocation));
4549
- }
4550
- waitForInitialization(call, initialization) {
4551
- return new Promise((resolve, reject) => {
4552
- let timer;
4553
- let settled = false;
4554
- const deadline = typeof call.getDeadline === "function" ? call.getDeadline() : Infinity;
4555
- const deadlineMs = deadline instanceof Date ? deadline.getTime() : Number(deadline);
4556
- const cancellationError = () => Number.isFinite(deadlineMs) && deadlineMs <= Date.now() ? rpcError(
4557
- grpc.status.DEADLINE_EXCEEDED,
4558
- "Initialization did not finish before the allocation deadline"
4559
- ) : rpcError(
4560
- grpc.status.CANCELLED,
4561
- "Create allocation RPC was cancelled during initialization"
4562
- );
4563
- const onCancelled = () => finish(cancellationError());
4564
- const finish = (error) => {
4565
- if (settled) return;
4566
- settled = true;
4567
- if (timer != null) clearTimeout(timer);
4568
- if (typeof call.off === "function") call.off("cancelled", onCancelled);
4569
- if (error == null) resolve();
4570
- else reject(error);
4571
- };
4572
- if (call.cancelled || Number.isFinite(deadlineMs) && deadlineMs <= Date.now()) {
4573
- finish(cancellationError());
4574
- return;
4575
- }
4576
- if (typeof call.once === "function") call.once("cancelled", onCancelled);
4577
- if (Number.isFinite(deadlineMs)) {
4578
- const remainingMs = deadlineMs - Date.now();
4579
- if (remainingMs <= 0) {
4580
- finish(rpcError(grpc.status.DEADLINE_EXCEEDED, "Initialization did not finish before the allocation deadline"));
4581
- return;
4582
- }
4583
- timer = setTimeout(() => {
4584
- finish(rpcError(
4585
- grpc.status.DEADLINE_EXCEEDED,
4586
- "Initialization did not finish before the allocation deadline"
4587
- ));
4588
- }, remainingMs);
4589
- }
4590
- initialization.then(
4591
- () => finish(
4592
- call.cancelled || Number.isFinite(deadlineMs) && deadlineMs <= Date.now() ? cancellationError() : void 0
4593
- ),
4594
- () => finish(rpcError(
4595
- grpc.status.FAILED_PRECONDITION,
4596
- "Function Executor initialization failed"
4597
- ))
4598
- );
4599
- });
4600
- }
4601
- assertAllocationAdmissionActive(call) {
4602
- const deadline = typeof call.getDeadline === "function" ? call.getDeadline() : Infinity;
4603
- const deadlineMs = deadline instanceof Date ? deadline.getTime() : Number(deadline);
4604
- if (Number.isFinite(deadlineMs) && deadlineMs <= Date.now()) {
4605
- throw rpcError(grpc.status.DEADLINE_EXCEEDED, "Create allocation RPC deadline exceeded");
4606
- }
4607
- if (call.cancelled) {
4608
- throw rpcError(grpc.status.CANCELLED, "Create allocation RPC was cancelled before allocation start");
4609
- }
4610
- }
4611
- deleteAllocation(call, callback) {
4612
- try {
4613
- const id = String(call.request.allocationId ?? "");
4614
- const allocation = this.getAllocation(id);
4615
- this.log("debug", "delete allocation RPC received", {
4616
- allocation_id: id,
4617
- finished: allocation.isFinished
4618
- });
4619
- if (!allocation.isFinished) {
4620
- throw rpcError(grpc.status.FAILED_PRECONDITION, `Allocation ${id} is still running`);
4621
- }
4622
- this.allocations.delete(id);
4623
- this.log("info", "allocation deleted", { allocation_id: id });
4624
- callback(null, {});
4625
- } catch (error) {
4626
- this.log("error", "delete allocation RPC failed", { allocation_id: call.request.allocationId }, error);
4627
- callback(error);
4628
- }
4629
- }
4630
- watchAllocationState(call) {
4631
- try {
4632
- const allocationId = String(call.request.allocationId ?? "");
4633
- this.log("debug", "allocation state stream opening", { allocation_id: allocationId });
4634
- this.getAllocation(allocationId).watchState(call);
4635
- } catch (error) {
4636
- this.log("error", "allocation state stream failed to open", { allocation_id: call.request.allocationId }, error);
4637
- call.destroy(error);
4638
- }
4639
- }
4640
- sendAllocationUpdate(call, callback) {
4641
- try {
4642
- const allocationId = String(call.request.allocationId ?? "");
4643
- this.log("debug", "allocation update RPC received", {
4644
- allocation_id: allocationId,
4645
- update_kind: call.request.outputBlob != null ? "output_blob" : call.request.requestStateOperationResult != null ? "request_state_operation_result" : "unknown"
4646
- });
4647
- if (call.request.outputBlob == null && call.request.requestStateOperationResult == null) {
4648
- throw rpcError(
4649
- grpc.status.INVALID_ARGUMENT,
4650
- `Unknown update type in AllocationUpdate for allocation ${allocationId}`
4651
- );
4652
- }
4653
- const allocation = this.getAllocation(allocationId);
4654
- if (allocation.isFinished) throw rpcError(grpc.status.FAILED_PRECONDITION, "Allocation is already finished");
4655
- allocation.deliverUpdate(call.request);
4656
- callback(null, {});
4657
- } catch (error) {
4658
- this.log("error", "allocation update RPC failed", { allocation_id: call.request.allocationId }, error);
4659
- callback(error);
4660
- }
4661
- }
4662
- getAllocationExecutionLogBatch(call, callback) {
4663
- try {
4664
- const allocationId = String(call.request.allocationId ?? "");
4665
- this.log("debug", "execution log batch RPC waiting", { allocation_id: allocationId });
4666
- const allocation = this.getAllocation(allocationId);
4667
- void allocation.getExecutionBatch().then(
4668
- (events) => {
4669
- this.log("debug", "execution log batch RPC completed", {
4670
- allocation_id: allocationId,
4671
- event_count: events.length
4672
- });
4673
- callback(null, { events });
4674
- },
4675
- (error) => {
4676
- this.log("error", "execution log batch RPC failed", { allocation_id: allocationId }, error);
4677
- callback(error);
4678
- }
4679
- );
4680
- } catch (error) {
4681
- this.log("error", "execution log batch RPC failed", { allocation_id: call.request.allocationId }, error);
4682
- callback(error);
4683
- }
4684
- }
4685
- advanceAllocationExecutionLogBatch(call, callback) {
4686
- try {
4687
- const allocationId = String(call.request.allocationId ?? "");
4688
- this.log("debug", "advance execution log batch RPC received", { allocation_id: allocationId });
4689
- this.getAllocation(allocationId).advanceExecutionBatch();
4690
- callback(null, {});
4691
- } catch (error) {
4692
- this.log("error", "advance execution log batch RPC failed", { allocation_id: call.request.allocationId }, error);
4693
- callback(error);
4694
- }
4695
- }
4696
- watchAllocationEventLogReads(call) {
4697
- try {
4698
- const allocationId = String(call.request.allocationId ?? "");
4699
- const allocation = this.getAllocation(allocationId);
4700
- this.log("debug", "event log read stream opening", { allocation_id: allocationId });
4701
- call.sendMetadata(new grpc.Metadata());
4702
- this.log("debug", "event log read stream established", { allocation_id: allocationId });
4703
- allocation.watchEventLogReads(call);
4704
- } catch (error) {
4705
- this.log("error", "event log read stream failed to open", { allocation_id: call.request.allocationId }, error);
4706
- call.destroy(error);
4707
- }
4708
- }
4709
- sendAllocationEventLogReadResponse(call, callback) {
4710
- try {
4711
- const allocationId = String(call.request.allocationId ?? "");
4712
- this.log("debug", "event log read response RPC received", {
4713
- allocation_id: allocationId,
4714
- entry_count: call.request.entries?.length ?? 0,
4715
- last_clock: call.request.lastClock,
4716
- has_more: call.request.hasMore
4717
- });
4718
- this.getAllocation(allocationId).deliverEventLogResponse(call.request);
4719
- callback(null, {});
4720
- } catch (error) {
4721
- this.log("error", "event log read response RPC failed", { allocation_id: call.request.allocationId }, error);
4722
- callback(error);
4723
- }
4724
- }
4725
- checkHealth(_call, callback) {
4726
- this.log("debug", "health check RPC received", { initialized: this.definition != null });
4727
- if (this.definition == null) {
4728
- callback(rpcError(grpc.status.UNAVAILABLE, "Function Executor is not initialized"));
4729
- } else {
4730
- callback(null, { healthy: true, statusMessage: "ok" });
4731
- }
4732
- }
4733
- getInfo(_call, callback) {
4734
- this.log("debug", "executor info RPC received", { protocol_version: "0.1.3" });
4735
- callback(null, {
4736
- version: "0.1.3",
4737
- sdkVersion: SDK_VERSION,
4738
- sdkLanguage: "typescript",
4739
- sdkLanguageVersion: process.versions.node
4740
- });
4741
- }
4742
- functionLogFields(functionRef) {
4743
- return {
4744
- namespace: functionRef?.namespace,
4745
- application: functionRef?.applicationName,
4746
- application_version: functionRef?.applicationVersion,
4747
- function: functionRef?.functionName
4748
- };
4749
- }
4750
- allocationLogFields(allocation) {
4751
- return {
4752
- allocation_id: allocation?.allocationId,
4753
- request_id: allocation?.requestId,
4754
- function_call_id: allocation?.functionCallId
4755
- };
4756
- }
4757
- log(level, message, fields = {}, error) {
4758
- writeStructuredOutput("stderr", () => {
4759
- const rendered = error instanceof Error ? error.stack?.split("\n") ?? [`${error.name}: ${error.message}`] : error == null ? void 0 : [String(error)];
4760
- return {
4761
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4762
- level,
4763
- component: "typescript_function_executor_service",
4764
- message,
4765
- ...fields,
4766
- ...rendered == null ? {} : { error: rendered }
4767
- };
4768
- });
4769
- }
4770
- };
4771
-
4772
- // src/function-executor/main.ts
4773
- var SHUTDOWN_GRACE_PERIOD_MS = 1e4;
4774
- var IPV4_LOOPBACK_ADDRESS = "127.0.0.1";
4775
- function functionExecutorBindAddress(address) {
4776
- if (address === "localhost") return IPV4_LOOPBACK_ADDRESS;
4777
- if (address.startsWith("localhost:")) {
4778
- return `${IPV4_LOOPBACK_ADDRESS}${address.slice("localhost".length)}`;
4779
- }
4780
- return address;
4781
- }
4782
- function parseArguments(argv) {
4783
- const result = { functionExecutorId: "" };
4784
- for (let index = 0; index < argv.length; index += 1) {
4785
- const argument = argv[index];
4786
- const [key, inline] = argument.split("=", 2);
4787
- const value = inline ?? argv[index + 1];
4788
- if (key === "--address") result.address = value;
4789
- else if (key === "--executor-id") result.executorId = value;
4790
- else if (key === "--function-executor-id") result.functionExecutorId = value ?? "";
4791
- else continue;
4792
- if (inline == null) index += 1;
4793
- }
4794
- return result;
4795
- }
4796
- function serviceDefinition() {
4797
- const protoRoot = functionExecutorProtoRoot();
4798
- const protoPath = import_node_path2.default.join(protoRoot, "tensorlake/function_executor/proto/function_executor.proto");
4799
- const packageDefinition = (0, import_proto_loader.loadSync)(protoPath, {
4800
- includeDirs: [import_node_path2.default.dirname(protoPath), protoRoot, (0, import_google_proto_files.getProtoPath)()],
4801
- keepCase: false,
4802
- longs: Number,
4803
- enums: String,
4804
- defaults: false,
4805
- oneofs: true
4806
- });
4807
- const loaded = grpc2.loadPackageDefinition(packageDefinition);
4808
- return loaded.function_executor_service.FunctionExecutor.service;
4809
- }
4810
- function functionExecutorProtoRoot() {
4811
- const relativeRoots = [
4812
- // Standalone function-executor capsule.
4813
- "../../proto/",
4814
- // Function executor launched directly from the published SDK package.
4815
- "../../runtime/function-executor/package/proto/",
4816
- // Development checkout: both language runtimes use the repository source.
4817
- "../../../proto/"
4818
- ];
4819
- for (const relativeRoot of relativeRoots) {
4820
- const protoRoot = (0, import_node_url3.fileURLToPath)(new URL(relativeRoot, importMetaUrl));
4821
- const protoPath = import_node_path2.default.join(protoRoot, "tensorlake/function_executor/proto/function_executor.proto");
4822
- if ((0, import_node_fs.existsSync)(protoPath)) return protoRoot;
4823
- }
4824
- throw new Error("Cannot find the shared Tensorlake function-executor protobuf definition");
4825
- }
4826
- async function main(argv = process.argv.slice(2)) {
4827
- configureNativeWorker(() => new URL("../native-worker.cjs", importMetaUrl));
4828
- const nodeMajor = Number(process.versions.node.split(".")[0]);
4829
- if (nodeMajor < 24) throw new Error(`Tensorlake TypeScript functions require Node 24 or newer; got ${process.versions.node}`);
4830
- const args = parseArguments(argv);
4831
- if (!args.address) throw new Error("--address argument is required");
4832
- if (!args.executorId) throw new Error("--executor-id argument is required");
4833
- const address = functionExecutorBindAddress(args.address);
4834
- writeStructuredOutput("stderr", () => ({
4835
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4836
- level: "info",
4837
- component: "typescript_function_executor_main",
4838
- message: "starting TypeScript function executor",
4839
- address,
4840
- executor_id: args.executorId,
4841
- fn_executor_id: args.functionExecutorId,
4842
- node_version: process.versions.node,
4843
- pid: process.pid
4844
- }));
4845
- const server = new grpc2.Server({
4846
- "grpc.max_receive_message_length": -1,
4847
- "grpc.max_send_message_length": -1,
4848
- "grpc.so_reuseport": 0
4849
- });
4850
- const service = new FunctionExecutorService();
4851
- server.addService(serviceDefinition(), service.implementation);
4852
- await new Promise((resolve, reject) => {
4853
- server.bindAsync(address, grpc2.ServerCredentials.createInsecure(), (error) => {
4854
- if (error != null) reject(error);
4855
- else resolve();
4856
- });
4857
- });
4858
- writeStructuredOutput("stderr", () => ({
4859
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4860
- level: "info",
4861
- component: "typescript_function_executor_main",
4862
- message: "started TypeScript function executor",
4863
- address,
4864
- executor_id: args.executorId,
4865
- fn_executor_id: args.functionExecutorId,
4866
- node_version: process.versions.node
4867
- }));
4868
- let stopping = false;
4869
- const shutdown = (signal) => {
4870
- if (stopping) return;
4871
- stopping = true;
4872
- writeStructuredOutput("stderr", () => ({
4873
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4874
- level: "info",
4875
- component: "typescript_function_executor_main",
4876
- message: "stopping TypeScript function executor",
4877
- signal,
4878
- executor_id: args.executorId,
4879
- fn_executor_id: args.functionExecutorId,
4880
- pid: process.pid
4881
- }));
4882
- const forceShutdownTimer = setTimeout(() => {
4883
- writeStructuredOutput("stderr", () => ({
4884
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4885
- level: "error",
4886
- component: "typescript_function_executor_main",
4887
- message: "forcing TypeScript function executor shutdown after grace period",
4888
- signal,
4889
- executor_id: args.executorId,
4890
- fn_executor_id: args.functionExecutorId,
4891
- pid: process.pid,
4892
- grace_period_ms: SHUTDOWN_GRACE_PERIOD_MS
4893
- }));
4894
- server.forceShutdown();
4895
- process.exit(1);
4896
- }, SHUTDOWN_GRACE_PERIOD_MS);
4897
- const gracefulShutdown = service.shutdown(new Error(`Function executor received ${signal}`)).then(() => new Promise((resolve, reject) => {
4898
- server.tryShutdown((error) => {
4899
- if (error != null) reject(error);
4900
- else resolve();
4901
- });
4902
- }));
4903
- void gracefulShutdown.then(
4904
- () => {
4905
- clearTimeout(forceShutdownTimer);
4906
- writeStructuredOutput("stderr", () => ({
4907
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4908
- level: "info",
4909
- component: "typescript_function_executor_main",
4910
- message: "stopped TypeScript function executor",
4911
- signal,
4912
- executor_id: args.executorId,
4913
- fn_executor_id: args.functionExecutorId,
4914
- pid: process.pid
4915
- }));
4916
- process.exit(0);
4917
- },
4918
- (error) => {
4919
- clearTimeout(forceShutdownTimer);
4920
- writeStructuredOutput("stderr", () => ({
4921
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4922
- level: "error",
4923
- component: "typescript_function_executor_main",
4924
- message: "TypeScript function executor shutdown failed",
4925
- signal,
4926
- executor_id: args.executorId,
4927
- fn_executor_id: args.functionExecutorId,
4928
- pid: process.pid,
4929
- error: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
4930
- }));
4931
- server.forceShutdown();
4932
- process.exit(1);
4933
- }
4934
- );
4935
- };
4936
- process.once("SIGINT", () => shutdown("SIGINT"));
4937
- process.once("SIGTERM", () => shutdown("SIGTERM"));
4938
- }
4939
- // Annotate the CommonJS export names for ESM import in node:
4940
- 0 && (module.exports = {
4941
- functionExecutorBindAddress,
4942
- main
4943
- });
4944
- //# sourceMappingURL=main.cjs.map