tensorlake 0.5.143 → 0.5.148

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +11 -159
  2. package/applications.cjs +3 -0
  3. package/applications.d.cts +1 -0
  4. package/applications.d.ts +1 -0
  5. package/applications.js +1 -0
  6. package/index.cjs +3 -0
  7. package/index.d.cts +1 -0
  8. package/index.d.ts +1 -0
  9. package/index.js +1 -0
  10. package/package.json +25 -68
  11. package/bin/function-executor.js +0 -8
  12. package/bin/tensorlake-create-sandbox-image.js +0 -9
  13. package/bin/tensorlake-deploy.cjs +0 -8
  14. package/bin/tensorlake-import-sandbox-image.js +0 -9
  15. package/bin/tensorlake-typescript-function-runner.js +0 -8
  16. package/dist/applications/index.cjs +0 -4821
  17. package/dist/applications/index.cjs.map +0 -1
  18. package/dist/applications/index.d.cts +0 -130
  19. package/dist/applications/index.d.ts +0 -130
  20. package/dist/applications/index.js +0 -4782
  21. package/dist/applications/index.js.map +0 -1
  22. package/dist/function-agent/main.cjs +0 -1838
  23. package/dist/function-agent/main.cjs.map +0 -1
  24. package/dist/function-agent/main.d.cts +0 -3
  25. package/dist/function-agent/main.d.ts +0 -3
  26. package/dist/function-agent/main.js +0 -1825
  27. package/dist/function-agent/main.js.map +0 -1
  28. package/dist/function-executor/main.cjs +0 -4944
  29. package/dist/function-executor/main.cjs.map +0 -1
  30. package/dist/function-executor/main.d.cts +0 -4
  31. package/dist/function-executor/main.d.ts +0 -4
  32. package/dist/function-executor/main.js +0 -4902
  33. package/dist/function-executor/main.js.map +0 -1
  34. package/dist/index.cjs +0 -10184
  35. package/dist/index.cjs.map +0 -1
  36. package/dist/index.d.cts +0 -2431
  37. package/dist/index.d.ts +0 -2431
  38. package/dist/index.js +0 -10108
  39. package/dist/index.js.map +0 -1
  40. package/dist/native-worker.cjs +0 -278
  41. package/dist/native-worker.cjs.map +0 -1
  42. package/dist/native-worker.d.cts +0 -2
  43. package/dist/native-worker.d.ts +0 -2
  44. package/dist/native-worker.js +0 -276
  45. package/dist/native-worker.js.map +0 -1
  46. package/dist/remote-CNJ9TmcQ.d.cts +0 -283
  47. package/dist/remote-dblFxKug.d.ts +0 -283
  48. package/dist/sandbox-image-ChzWDeZy.d.cts +0 -357
  49. package/dist/sandbox-image-ChzWDeZy.d.ts +0 -357
  50. package/dist/sandbox-image.cjs +0 -1992
  51. package/dist/sandbox-image.cjs.map +0 -1
  52. package/dist/sandbox-image.d.cts +0 -1
  53. package/dist/sandbox-image.d.ts +0 -1
  54. package/dist/sandbox-image.js +0 -1987
  55. package/dist/sandbox-image.js.map +0 -1
  56. package/lib/libc.cjs +0 -61
  57. package/lib/libc.d.cts +0 -1
  58. package/lib/runtime.cjs +0 -155
  59. package/runtime/function-executor/manifest.json +0 -59
  60. package/runtime/function-executor/package/bin/function-executor.js +0 -8
  61. package/runtime/function-executor/package/dist/function-executor/main.js +0 -4902
  62. package/runtime/function-executor/package/dist/function-executor/main.js.map +0 -1
  63. package/runtime/function-executor/package/dist/native-worker.cjs +0 -278
  64. package/runtime/function-executor/package/dist/native-worker.cjs.map +0 -1
  65. package/runtime/function-executor/package/npm-shrinkwrap.json +0 -475
  66. package/runtime/function-executor/package/package.json +0 -22
  67. package/runtime/function-executor/package/proto/README.md +0 -18
  68. package/runtime/function-executor/package/proto/tensorlake/function_executor/proto/function_executor.proto +0 -613
  69. package/runtime/function-executor/package/proto/tensorlake/function_executor/proto/status.proto +0 -42
  70. package/runtime/typescript-function-runner/manifest.json +0 -44
  71. package/runtime/typescript-function-runner/package/bin/tensorlake-typescript-function-runner.js +0 -8
  72. package/runtime/typescript-function-runner/package/dist/function-agent/main.js +0 -1825
  73. package/runtime/typescript-function-runner/package/dist/function-agent/main.js.map +0 -1
  74. package/runtime/typescript-function-runner/package/dist/native-worker.cjs +0 -278
  75. package/runtime/typescript-function-runner/package/dist/native-worker.cjs.map +0 -1
  76. package/runtime/typescript-function-runner/package/npm-shrinkwrap.json +0 -580
  77. package/runtime/typescript-function-runner/package/package.json +0 -25
@@ -1,4902 +0,0 @@
1
- // src/function-executor/main.ts
2
- import { existsSync } from "fs";
3
-
4
- // src/native-worker-client.ts
5
- import { createRequire } from "module";
6
- import { Worker } from "worker_threads";
7
-
8
- // src/native-worker-protocol.ts
9
- var cancelNativeCall = /* @__PURE__ */ Symbol.for("tensorlake.native.cancel.v1");
10
- var closeNativeHandle = /* @__PURE__ */ Symbol.for("tensorlake.native.close.v1");
11
-
12
- // src/defaults.ts
13
- var SDK_VERSION = "0.5.143";
14
- var API_URL = process.env.TENSORLAKE_API_URL ?? "https://api.tensorlake.ai";
15
- var API_KEY = process.env.TENSORLAKE_API_KEY ?? void 0;
16
- var NAMESPACE = process.env.INDEXIFY_NAMESPACE ?? "default";
17
- var SANDBOX_PROXY_URL = process.env.TENSORLAKE_SANDBOX_PROXY_URL ?? "https://sandbox.tensorlake.ai";
18
-
19
- // src/url.ts
20
- function trimTrailingSlashes(url) {
21
- return url.endsWith("/") ? url.slice(0, -1) : url;
22
- }
23
- function resolveProxyTarget(proxyUrl, sandboxId) {
24
- try {
25
- const parsed = new URL(proxyUrl);
26
- const host = parsed.hostname;
27
- if (host === "localhost" || host === "127.0.0.1") {
28
- return {
29
- baseUrl: trimTrailingSlashes(proxyUrl),
30
- hostHeader: `${sandboxId}.local`,
31
- sandboxIdHeader: void 0
32
- };
33
- }
34
- const port = parsed.port ? `:${parsed.port}` : "";
35
- return {
36
- baseUrl: `${parsed.protocol}//${host}${port}`,
37
- hostHeader: void 0,
38
- sandboxIdHeader: sandboxId
39
- };
40
- } catch {
41
- return {
42
- baseUrl: `${trimTrailingSlashes(proxyUrl)}/${sandboxId}`,
43
- hostHeader: void 0,
44
- sandboxIdHeader: void 0
45
- };
46
- }
47
- }
48
-
49
- // src/native-worker-client.ts
50
- var NativeWorkerClient = class {
51
- constructor(workerPath = () => createRequire(import.meta.url).resolve(
52
- "tensorlake/internal/native-worker"
53
- )) {
54
- this.workerPath = workerPath;
55
- }
56
- worker;
57
- nextId = 1;
58
- registered = /* @__PURE__ */ new Set();
59
- pending = /* @__PURE__ */ new Map();
60
- finalizers = new FinalizationRegistry(
61
- (id) => this.release(id)
62
- );
63
- handle(kind, args, parent) {
64
- const handle = { id: this.nextId++, kind, args, parent };
65
- this.finalizers.register(handle, handle.id);
66
- let closed = false;
67
- return new Proxy(
68
- {},
69
- {
70
- get: (_target, method) => {
71
- if (method === "then") return void 0;
72
- if (method === closeNativeHandle)
73
- return () => {
74
- closed = true;
75
- this.release(handle.id);
76
- };
77
- if (method === "connectProxy")
78
- return (...proxyArgs) => {
79
- if (closed) throw new Error("Tensorlake native client is closed");
80
- return this.handle("NativeSandboxProxyClient", proxyArgs, handle);
81
- };
82
- if (method === "baseUrl")
83
- return () => resolveProxyTarget(String(args[0]), String(args[1])).baseUrl;
84
- if (typeof method !== "string") return void 0;
85
- return (...callArgs) => {
86
- if (closed)
87
- return Promise.reject(
88
- new Error("Tensorlake native client is closed")
89
- );
90
- const emit = typeof callArgs.at(-1) === "function" ? callArgs.pop() : void 0;
91
- return this.call(handle, method, callArgs, emit);
92
- };
93
- }
94
- }
95
- );
96
- }
97
- call(handle, method, args, emit) {
98
- const id = this.nextId++;
99
- let worker;
100
- const promise = new Promise((resolve, reject) => {
101
- try {
102
- worker = this.getWorker();
103
- this.pending.set(id, { resolve, reject, emit, handle });
104
- worker.ref();
105
- if (handle) this.register(handle, worker);
106
- this.send(worker, {
107
- type: "call",
108
- id,
109
- handleId: handle?.id,
110
- method,
111
- args,
112
- stream: !!emit
113
- });
114
- } catch (error) {
115
- this.pending.delete(id);
116
- this.unrefIfIdle();
117
- reject(error);
118
- }
119
- });
120
- promise[cancelNativeCall] = () => {
121
- const pending = this.pending.get(id);
122
- if (pending && worker) {
123
- pending.emit = void 0;
124
- this.send(worker, { type: "cancel", id });
125
- }
126
- };
127
- return promise;
128
- }
129
- getWorker() {
130
- if (this.worker) return this.worker;
131
- const worker = new Worker(this.workerPath(), {
132
- // The bundled worker needs no application loaders or telemetry preloads.
133
- // NODE_OPTIONS is parsed independently of execArgv by Node.
134
- execArgv: [],
135
- env: { ...process.env, NODE_OPTIONS: "" },
136
- name: "tensorlake-native"
137
- });
138
- this.worker = worker;
139
- worker.on("message", (message) => {
140
- const pending = this.pending.get(message.id);
141
- if (!pending || this.worker !== worker) return;
142
- if (message.type === "event") {
143
- Promise.resolve().then(() => pending.emit?.(message.value)).then(
144
- () => {
145
- if (this.worker === worker)
146
- this.send(worker, { type: "ack", id: message.id });
147
- },
148
- (error) => {
149
- this.send(worker, { type: "cancel", id: message.id });
150
- pending.reject(error);
151
- }
152
- );
153
- return;
154
- }
155
- this.pending.delete(message.id);
156
- if (message.type === "error") {
157
- const error = new Error(message.error.message);
158
- error.name = message.error.name;
159
- error.stack = message.error.stack ?? error.stack;
160
- pending.reject(error);
161
- } else {
162
- pending.resolve(message.value);
163
- }
164
- this.unrefIfIdle();
165
- });
166
- const fail = (error) => {
167
- if (this.worker !== worker) return;
168
- this.worker = void 0;
169
- this.registered.clear();
170
- for (const pending of this.pending.values()) pending.reject(error);
171
- this.pending.clear();
172
- void worker.terminate();
173
- };
174
- worker.on("error", fail);
175
- worker.on("messageerror", fail);
176
- worker.on(
177
- "exit",
178
- (code) => fail(
179
- new Error(
180
- `Tensorlake native worker exited (${code}); in-flight operations were not retried`
181
- )
182
- )
183
- );
184
- worker.unref();
185
- return worker;
186
- }
187
- register(handle, worker) {
188
- if (this.registered.has(handle.id)) return;
189
- if (handle.parent) this.register(handle.parent, worker);
190
- this.send(worker, {
191
- type: "create",
192
- handle: { id: handle.id, kind: handle.kind, args: handle.args },
193
- parentId: handle.parent?.id
194
- });
195
- this.registered.add(handle.id);
196
- }
197
- release(handleId) {
198
- if (this.worker && this.registered.delete(handleId))
199
- this.send(this.worker, { type: "release", handleId });
200
- }
201
- send(worker, message) {
202
- worker.postMessage(message);
203
- }
204
- unrefIfIdle() {
205
- if (this.pending.size === 0) this.worker?.unref();
206
- }
207
- };
208
- var runtimeKey = /* @__PURE__ */ Symbol.for(
209
- `tensorlake.native.worker.${false ? "source" : "0.5.143"}.v1`
210
- );
211
- var runtimes = globalThis;
212
- function configureNativeWorker(workerPath) {
213
- runtimes[runtimeKey] ??= new NativeWorkerClient(workerPath);
214
- }
215
-
216
- // src/function-executor/main.ts
217
- import path2 from "path";
218
- import { fileURLToPath as fileURLToPath2 } from "url";
219
- import * as grpc2 from "@grpc/grpc-js";
220
- import { loadSync } from "@grpc/proto-loader";
221
- import { getProtoPath } from "google-proto-files";
222
-
223
- // src/function-executor/safe-output.ts
224
- var nativeJSONStringify = JSON.stringify;
225
- function captureStream(stream) {
226
- stream.on("error", () => void 0);
227
- return {
228
- // Deployed code runs in this process and can replace stream.write. Capture
229
- // the executor's writer before application modules are imported.
230
- write: stream.write.bind(stream)
231
- };
232
- }
233
- var executorOutput = {
234
- stdout: captureStream(process.stdout),
235
- stderr: captureStream(process.stderr)
236
- };
237
- function writeStructuredOutput(destination, createRecord) {
238
- try {
239
- const serialized = nativeJSONStringify(createRecord());
240
- if (serialized == null) return;
241
- executorOutput[destination].write(`${serialized}
242
- `, () => void 0);
243
- } catch {
244
- }
245
- }
246
-
247
- // src/function-executor/service.ts
248
- import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises";
249
- import os from "os";
250
- import path from "path";
251
- import { pathToFileURL } from "url";
252
- import { compileFunction, constants as vmConstants } from "vm";
253
- import * as grpc from "@grpc/grpc-js";
254
- import { unzipSync } from "fflate";
255
-
256
- // src/applications/errors.ts
257
- var SDK_USAGE_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.sdk-usage-error.v1");
258
- var REQUEST_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.request-error.v1");
259
- var FUNCTION_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.function-error.v1");
260
- var TIMEOUT_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.timeout-error.v1");
261
- var TensorlakeApplicationError = class extends Error {
262
- constructor(message, options) {
263
- super(message, options);
264
- this.name = "TensorlakeApplicationError";
265
- }
266
- };
267
- var SDKUsageError = class _SDKUsageError extends TensorlakeApplicationError {
268
- [SDK_USAGE_ERROR_BRAND] = true;
269
- static [Symbol.hasInstance](value) {
270
- if (this !== _SDKUsageError) {
271
- return Function.prototype[Symbol.hasInstance].call(this, value);
272
- }
273
- return isSDKUsageError(value);
274
- }
275
- constructor(message) {
276
- super(message);
277
- this.name = "SDKUsageError";
278
- }
279
- };
280
- function isSDKUsageError(value) {
281
- if (typeof value !== "object" || value == null) return false;
282
- const candidate = value;
283
- return candidate[SDK_USAGE_ERROR_BRAND] === true && typeof candidate.message === "string";
284
- }
285
- var SerializationError = class extends TensorlakeApplicationError {
286
- constructor(message, options) {
287
- super(message, options);
288
- this.name = "SerializationError";
289
- }
290
- };
291
- var DeserializationError = class extends TensorlakeApplicationError {
292
- constructor(message, options) {
293
- super(message, options);
294
- this.name = "DeserializationError";
295
- }
296
- };
297
- var FunctionError = class _FunctionError extends TensorlakeApplicationError {
298
- [FUNCTION_ERROR_BRAND] = true;
299
- static [Symbol.hasInstance](value) {
300
- if (this !== _FunctionError) {
301
- return Function.prototype[Symbol.hasInstance].call(this, value);
302
- }
303
- return isFunctionError(value);
304
- }
305
- constructor(message, options) {
306
- super(message, options);
307
- this.name = "FunctionError";
308
- }
309
- };
310
- var TimeoutError = class _TimeoutError extends TensorlakeApplicationError {
311
- [TIMEOUT_ERROR_BRAND] = true;
312
- static [Symbol.hasInstance](value) {
313
- if (this !== _TimeoutError) {
314
- return Function.prototype[Symbol.hasInstance].call(this, value);
315
- }
316
- return isTimeoutError(value);
317
- }
318
- constructor() {
319
- super("Timed out.");
320
- this.name = "TimeoutError";
321
- }
322
- };
323
- function isTimeoutError(value) {
324
- if (typeof value !== "object" || value == null) return false;
325
- const candidate = value;
326
- return candidate[TIMEOUT_ERROR_BRAND] === true && typeof candidate.message === "string";
327
- }
328
- function isFunctionError(value) {
329
- if (typeof value !== "object" || value == null) return false;
330
- const candidate = value;
331
- return candidate[FUNCTION_ERROR_BRAND] === true && typeof candidate.message === "string";
332
- }
333
- var RequestError = class _RequestError extends TensorlakeApplicationError {
334
- [REQUEST_ERROR_BRAND] = true;
335
- static [Symbol.hasInstance](value) {
336
- if (this !== _RequestError) {
337
- return Function.prototype[Symbol.hasInstance].call(this, value);
338
- }
339
- return isRequestError(value);
340
- }
341
- constructor(message, options) {
342
- super(message, options);
343
- this.name = "RequestError";
344
- }
345
- };
346
- function isRequestError(value) {
347
- if (typeof value !== "object" || value == null) return false;
348
- const candidate = value;
349
- return candidate[REQUEST_ERROR_BRAND] === true && typeof candidate.message === "string";
350
- }
351
- var ReplayMismatchError = class extends TensorlakeApplicationError {
352
- constructor(message = "Function execution diverged from its durable event history") {
353
- super(message);
354
- this.name = "ReplayMismatchError";
355
- }
356
- };
357
-
358
- // src/applications/registry.ts
359
- var REGISTRY_KEY = /* @__PURE__ */ Symbol.for("tensorlake.applications.registry.v1");
360
- function registry() {
361
- const target = globalThis;
362
- target[REGISTRY_KEY] ??= { functions: /* @__PURE__ */ new Map(), applications: /* @__PURE__ */ new Map() };
363
- return target[REGISTRY_KEY];
364
- }
365
- function snapshotRegistry() {
366
- const state = registry();
367
- return {
368
- functions: new Map(state.functions),
369
- applications: new Map(state.applications)
370
- };
371
- }
372
- function restoreRegistry(snapshot) {
373
- const state = registry();
374
- state.functions.clear();
375
- state.applications.clear();
376
- for (const [name, definition] of snapshot.functions) state.functions.set(name, definition);
377
- for (const [name, definition] of snapshot.applications) state.applications.set(name, definition);
378
- }
379
-
380
- // src/function-executor/allocation.ts
381
- import { AsyncResource, createHook, executionAsyncId as executionAsyncId2 } from "async_hooks";
382
- import { createHash as createHash3, randomUUID as randomUUID3 } from "crypto";
383
-
384
- // src/applications/function.ts
385
- import { AsyncLocalStorage, executionAsyncId } from "async_hooks";
386
- import { randomUUID } from "crypto";
387
-
388
- // src/applications/schema.ts
389
- import Ajv from "ajv";
390
- import Ajv2020 from "ajv/dist/2020.js";
391
-
392
- // src/applications/file.ts
393
- var FILE_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.file.v1");
394
- var File = class _File {
395
- [FILE_BRAND] = true;
396
- content;
397
- contentType;
398
- static [Symbol.hasInstance](value) {
399
- if (this !== _File) {
400
- return Function.prototype[Symbol.hasInstance].call(this, value);
401
- }
402
- return isFile(value);
403
- }
404
- constructor(content, contentType) {
405
- if (!contentType.trim()) {
406
- throw new SDKUsageError("File contentType cannot be empty");
407
- }
408
- this.content = content instanceof Uint8Array ? content : new Uint8Array(content);
409
- this.contentType = contentType;
410
- }
411
- };
412
- function isFile(value) {
413
- if (typeof value !== "object" || value == null) return false;
414
- const candidate = value;
415
- return candidate[FILE_BRAND] === true && candidate.content instanceof Uint8Array && typeof candidate.contentType === "string" && candidate.contentType.trim().length > 0;
416
- }
417
-
418
- // src/applications/http-body.ts
419
- var HTTP_BODY_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.http-body.v1");
420
- var HttpBody = class _HttpBody {
421
- [HTTP_BODY_BRAND] = true;
422
- content;
423
- contentType;
424
- static [Symbol.hasInstance](value) {
425
- if (this !== _HttpBody) {
426
- return Function.prototype[Symbol.hasInstance].call(this, value);
427
- }
428
- return isHttpBody(value);
429
- }
430
- constructor(content, contentType) {
431
- this.content = content instanceof Uint8Array ? content : new Uint8Array(content);
432
- this.contentType = contentType;
433
- }
434
- text(encoding = "utf-8") {
435
- return new TextDecoder(encoding, { fatal: true }).decode(this.content);
436
- }
437
- json() {
438
- return JSON.parse(this.text());
439
- }
440
- };
441
- function isHttpBody(value) {
442
- if (typeof value !== "object" || value == null) return false;
443
- const candidate = value;
444
- return candidate[HTTP_BODY_BRAND] === true && candidate.content instanceof Uint8Array && (candidate.contentType === void 0 || typeof candidate.contentType === "string");
445
- }
446
-
447
- // src/applications/schema.ts
448
- var ajvDraft7 = new Ajv({ allErrors: true, strict: false });
449
- var ajv2020 = new Ajv2020({ allErrors: true, strict: false });
450
- var validatorCache = /* @__PURE__ */ new WeakMap();
451
- function compileValidator(jsonSchema) {
452
- const dialect = jsonSchema.$schema;
453
- const engine = typeof dialect === "string" && /\/draft-07\/schema#?$/.test(dialect) ? ajvDraft7 : ajv2020;
454
- const validate = engine.compile(jsonSchema);
455
- return {
456
- validate,
457
- errorsText: (errors) => engine.errorsText(errors, { separator: "; " })
458
- };
459
- }
460
- function validateWithSchema(valueSchema, value, label) {
461
- if (valueSchema._file) {
462
- if (!isFile(value)) {
463
- throw new SDKUsageError(`${label} must be a Tensorlake File`);
464
- }
465
- return value;
466
- }
467
- if (valueSchema._httpBody) {
468
- if (!isHttpBody(value)) {
469
- throw new SDKUsageError(`${label} must be a Tensorlake HttpBody`);
470
- }
471
- return value;
472
- }
473
- if (isFile(value)) {
474
- throw new SDKUsageError(`${label} must use schema.file() for a Tensorlake File`);
475
- }
476
- if (isHttpBody(value)) {
477
- throw new SDKUsageError(
478
- `${label} must use schema.httpBody() for a Tensorlake HttpBody`
479
- );
480
- }
481
- let cached = validatorCache.get(valueSchema);
482
- if (cached == null) {
483
- try {
484
- cached = compileValidator(valueSchema.jsonSchema);
485
- } catch (error) {
486
- const details = error instanceof Error ? error.message : String(error);
487
- throw new SDKUsageError(`${label} uses an invalid JSON schema: ${details}`);
488
- }
489
- validatorCache.set(valueSchema, cached);
490
- }
491
- if (!cached.validate(value)) {
492
- const details = cached.errorsText(cached.validate.errors);
493
- throw new SDKUsageError(`${label} does not match its schema: ${details}`);
494
- }
495
- return value;
496
- }
497
-
498
- // src/applications/function.ts
499
- var TAIL_CALL_BRAND = /* @__PURE__ */ Symbol.for(
500
- "tensorlake.applications.tail-call.v1"
501
- );
502
- var DURABLE_PROMISE_OBSERVER = /* @__PURE__ */ Symbol.for(
503
- "tensorlake.applications.durable-promise-observer.v1"
504
- );
505
- var DURABLE_PROMISE_INTERNAL_THEN = /* @__PURE__ */ Symbol.for(
506
- "tensorlake.applications.durable-promise-internal-then.v1"
507
- );
508
- var FUNCTION_RUNTIME_STORAGE_KEY = /* @__PURE__ */ Symbol.for(
509
- "tensorlake.applications.function-runtime-storage.v1"
510
- );
511
- var PROMISE_INSTRUMENTED_KEY = /* @__PURE__ */ Symbol.for(
512
- "tensorlake.applications.promise-instrumented.v3"
513
- );
514
- var PROMISE_THEN_INSTRUMENTED_KEY = /* @__PURE__ */ Symbol.for(
515
- "tensorlake.applications.promise-then-instrumented.v1"
516
- );
517
- var DURABLE_THENABLE_REGISTRY_KEY = /* @__PURE__ */ Symbol.for(
518
- "tensorlake.applications.durable-thenable-registry.v2"
519
- );
520
- var INTERNAL_PROMISE_AGGREGATE_DEPTH_KEY = /* @__PURE__ */ Symbol.for(
521
- "tensorlake.applications.internal-promise-aggregate-depth.v1"
522
- );
523
- function runtimeStorage() {
524
- const target = globalThis;
525
- target[FUNCTION_RUNTIME_STORAGE_KEY] ??= new AsyncLocalStorage();
526
- return target[FUNCTION_RUNTIME_STORAGE_KEY];
527
- }
528
- function currentFunctionRuntime() {
529
- return runtimeStorage().getStore();
530
- }
531
- function runWithFunctionRuntime(runtime, callback) {
532
- return runtimeStorage().run(runtime, callback);
533
- }
534
- function isDurableThenable(value) {
535
- if (typeof value !== "object" && typeof value !== "function" || value == null) {
536
- return false;
537
- }
538
- return durableThenableRegistry().has(value);
539
- }
540
- function isDetachedDurableThenable(value) {
541
- const provenance = provenanceForDurableThenable(value);
542
- return provenance?.detached === true || provenance?.assimilation != null && !provenance.assimilation.active;
543
- }
544
- function durableThenableRegistry() {
545
- const target = globalThis;
546
- target[DURABLE_THENABLE_REGISTRY_KEY] ??= /* @__PURE__ */ new WeakMap();
547
- return target[DURABLE_THENABLE_REGISTRY_KEY];
548
- }
549
- function newDurableThenableMetadata() {
550
- return {
551
- aggregateExpectations: [],
552
- provenanceExpectations: []
553
- };
554
- }
555
- function metadataForDurableThenable(value) {
556
- return provenanceForDurableThenable(value)?.metadata;
557
- }
558
- function provenanceForDurableThenable(value) {
559
- if (typeof value !== "object" && typeof value !== "function" || value == null) {
560
- return void 0;
561
- }
562
- return durableThenableRegistry().get(value);
563
- }
564
- function registerDurableThenable(value, source) {
565
- const registry2 = durableThenableRegistry();
566
- const metadata = source == null ? registry2.get(value)?.metadata ?? newDurableThenableMetadata() : registry2.get(source)?.metadata ?? newDurableThenableMetadata();
567
- registry2.set(value, { metadata });
568
- if (source != null && !registry2.has(source)) registry2.set(source, { metadata });
569
- return value;
570
- }
571
- function registerDetachedDurableThenable(value) {
572
- const registry2 = durableThenableRegistry();
573
- registry2.set(value, {
574
- metadata: registry2.get(value)?.metadata ?? newDurableThenableMetadata(),
575
- detached: true
576
- });
577
- return value;
578
- }
579
- function inheritDurableThenable(value, source) {
580
- const provenance = durableThenableRegistry().get(source);
581
- if (provenance == null) return value;
582
- durableThenableRegistry().set(value, provenance);
583
- if (provenance.assimilation == null) {
584
- provenance.metadata.provenanceExpectations.push({
585
- value,
586
- creatorAsyncId: executionAsyncId()
587
- });
588
- }
589
- return value;
590
- }
591
- function registerDurableAssimilation(value, detach) {
592
- const metadata = metadataForDurableThenable(value);
593
- if (metadata == null) {
594
- return {
595
- finish: () => void 0,
596
- link: (linked) => linked
597
- };
598
- }
599
- const registration = {
600
- active: true,
601
- detach
602
- };
603
- const provenanceIndex = metadata.provenanceExpectations.findIndex(
604
- (expectation) => expectation.creatorAsyncId !== executionAsyncId()
605
- );
606
- if (provenanceIndex >= 0) {
607
- const [expectation] = metadata.provenanceExpectations.splice(provenanceIndex, 1);
608
- durableThenableRegistry().set(expectation.value, {
609
- metadata,
610
- assimilation: registration
611
- });
612
- }
613
- const expectationIndex = metadata.aggregateExpectations.findIndex(
614
- (expectation) => !expectation.scope.settled && expectation.creatorAsyncId !== executionAsyncId()
615
- );
616
- if (expectationIndex >= 0) {
617
- const [expectation] = metadata.aggregateExpectations.splice(expectationIndex, 1);
618
- associateAssimilationWithAggregate(registration, expectation.scope);
619
- }
620
- return {
621
- finish: () => finishDurableAssimilation(registration),
622
- link: (linked) => {
623
- durableThenableRegistry().set(linked, {
624
- metadata,
625
- assimilation: registration
626
- });
627
- return linked;
628
- }
629
- };
630
- }
631
- function associateAssimilationWithAggregate(registration, scope) {
632
- if (!registration.active || scope.settled) return;
633
- finishDurableAssimilation(registration);
634
- registration.detach();
635
- }
636
- function finishDurableAssimilation(registration) {
637
- if (!registration.active) return;
638
- registration.active = false;
639
- }
640
- function trackDurableAggregate(runtime, result, durableContenders, markResult) {
641
- const scope = {
642
- settled: false,
643
- metadata: /* @__PURE__ */ new Set()
644
- };
645
- if (markResult) {
646
- registerDetachedDurableThenable(result);
647
- }
648
- const creatorAsyncId = executionAsyncId();
649
- for (const contender of durableContenders) {
650
- const provenance = provenanceForDurableThenable(contender);
651
- if (provenance == null) continue;
652
- if (provenance.assimilation != null) {
653
- associateAssimilationWithAggregate(provenance.assimilation, scope);
654
- } else {
655
- provenance.metadata.aggregateExpectations.push({
656
- scope,
657
- creatorAsyncId
658
- });
659
- }
660
- if (scope.metadata.has(provenance.metadata)) continue;
661
- const { metadata } = provenance;
662
- scope.metadata.add(metadata);
663
- }
664
- const settle = () => {
665
- if (scope.settled) return;
666
- scope.settled = true;
667
- for (const metadata of scope.metadata) {
668
- for (let index = metadata.aggregateExpectations.length - 1; index >= 0; index -= 1) {
669
- if (metadata.aggregateExpectations[index].scope === scope) {
670
- metadata.aggregateExpectations.splice(index, 1);
671
- }
672
- }
673
- }
674
- scope.metadata.clear();
675
- };
676
- if (runtime.trackPromiseSettlement?.(result, settle) !== true) settle();
677
- }
678
- function promiseAggregateObservation(kind) {
679
- let settleLifetime;
680
- const ordinaryLifetime = new Promise((resolve) => {
681
- settleLifetime = resolve;
682
- });
683
- let contenderCount;
684
- let fulfillmentCount = 0;
685
- let rejectionCount = 0;
686
- let settled = false;
687
- const maybeSettle = () => {
688
- if (settled) return;
689
- const canNoLongerAdvance = kind === "any" ? contenderCount != null && rejectionCount === contenderCount : kind === "all" ? rejectionCount > 0 || contenderCount != null && fulfillmentCount === contenderCount : contenderCount != null && fulfillmentCount + rejectionCount === contenderCount;
690
- if (!canNoLongerAdvance) return;
691
- settled = true;
692
- settleLifetime();
693
- };
694
- return {
695
- ordinaryLifetime,
696
- allOrdinaryValuesObservable: true,
697
- observeFulfillment: () => {
698
- fulfillmentCount += 1;
699
- maybeSettle();
700
- },
701
- observeRejection: () => {
702
- rejectionCount += 1;
703
- maybeSettle();
704
- },
705
- finishIteration: (count) => {
706
- contenderCount = count;
707
- maybeSettle();
708
- }
709
- };
710
- }
711
- function observePromiseAggregateContender(normalized, observation) {
712
- if (typeof normalized !== "object" && typeof normalized !== "function" || normalized == null) {
713
- observation.allOrdinaryValuesObservable = false;
714
- return normalized;
715
- }
716
- let observedSettled = false;
717
- const observed = /* @__PURE__ */ Object.create(null);
718
- Object.defineProperty(observed, "then", {
719
- configurable: true,
720
- get: () => {
721
- const then = Reflect.get(normalized, "then", normalized);
722
- if (typeof then !== "function") return then;
723
- return function observedThen(onfulfilled, onrejected) {
724
- return Reflect.apply(then, normalized, [
725
- function observedFulfillment(value) {
726
- if (!observedSettled) {
727
- observedSettled = true;
728
- observation.observeFulfillment();
729
- }
730
- return typeof onfulfilled === "function" ? Reflect.apply(onfulfilled, this, [value]) : void 0;
731
- },
732
- function observedRejection(reason) {
733
- if (!observedSettled) {
734
- observedSettled = true;
735
- observation.observeRejection();
736
- }
737
- if (typeof onrejected === "function") {
738
- return Reflect.apply(onrejected, this, [reason]);
739
- }
740
- throw reason;
741
- }
742
- ]);
743
- };
744
- }
745
- });
746
- return observed;
747
- }
748
- function observedPromiseConstructor(constructor, observeNormalized) {
749
- const facade = function PromiseAggregateObservedConstructor(executor) {
750
- return Reflect.construct(
751
- constructor,
752
- [executor],
753
- constructor
754
- );
755
- };
756
- Object.defineProperty(facade, "resolve", {
757
- configurable: true,
758
- get: () => {
759
- const promiseResolve = Reflect.get(constructor, "resolve", constructor);
760
- if (typeof promiseResolve !== "function") return promiseResolve;
761
- return (value) => {
762
- const normalized = Reflect.apply(promiseResolve, constructor, [value]);
763
- return observeNormalized(normalized);
764
- };
765
- }
766
- });
767
- return facade;
768
- }
769
- function isInternalPromiseAggregate() {
770
- const target = globalThis;
771
- return (target[INTERNAL_PROMISE_AGGREGATE_DEPTH_KEY] ?? 0) > 0;
772
- }
773
- function installPromiseInstrumentation() {
774
- const target = globalThis;
775
- if (target[PROMISE_THEN_INSTRUMENTED_KEY] !== true) {
776
- const originalThen = Promise.prototype.then;
777
- Object.defineProperty(Promise.prototype, "then", {
778
- ...Object.getOwnPropertyDescriptor(Promise.prototype, "then"),
779
- value: function then(onfulfilled, onrejected) {
780
- const result = Reflect.apply(
781
- originalThen,
782
- this,
783
- [onfulfilled, onrejected]
784
- );
785
- if (isDetachedDurableThenable(this)) {
786
- currentFunctionRuntime()?.trackDurablePromiseConsumption?.(result);
787
- }
788
- return result;
789
- }
790
- });
791
- target[PROMISE_THEN_INSTRUMENTED_KEY] = true;
792
- }
793
- if (target[PROMISE_INSTRUMENTED_KEY] === true) return;
794
- const originalRace = Promise.race;
795
- const originalAny = Promise.any;
796
- const originalAll = Promise.all;
797
- const originalAllSettled = Promise.allSettled;
798
- const originalResolve = Promise.resolve;
799
- Object.defineProperty(Promise, "resolve", {
800
- ...Object.getOwnPropertyDescriptor(Promise, "resolve"),
801
- value: function resolve(value) {
802
- const result = Reflect.apply(originalResolve, this, [value]);
803
- if (isDurableThenable(value)) inheritDurableThenable(result, value);
804
- return result;
805
- }
806
- });
807
- Object.defineProperty(Promise, "race", {
808
- ...Object.getOwnPropertyDescriptor(Promise, "race"),
809
- value: function race(values) {
810
- const runtime = currentFunctionRuntime();
811
- if (runtime?.trackMixedPromiseRace == null) {
812
- return Reflect.apply(originalRace, this, [values]);
813
- }
814
- const durableContenders = [];
815
- let ordinaryContenderCount = 0;
816
- const trackedValues = {
817
- *[Symbol.iterator]() {
818
- for (const value of values) {
819
- yield value;
820
- }
821
- }
822
- };
823
- const observedConstructor = observedPromiseConstructor(
824
- this,
825
- (normalized) => {
826
- if (isDurableThenable(normalized)) {
827
- durableContenders.push(normalized);
828
- } else {
829
- ordinaryContenderCount += 1;
830
- }
831
- return normalized;
832
- }
833
- );
834
- const construct = () => Reflect.apply(
835
- originalRace,
836
- observedConstructor,
837
- [trackedValues]
838
- );
839
- const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
840
- if (durableContenders.length > 0) {
841
- trackDurableAggregate(
842
- runtime,
843
- result,
844
- durableContenders,
845
- !isInternalPromiseAggregate()
846
- );
847
- }
848
- if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
849
- runtime.trackMixedPromiseRace(result);
850
- }
851
- return result;
852
- }
853
- });
854
- Object.defineProperty(Promise, "any", {
855
- ...Object.getOwnPropertyDescriptor(Promise, "any"),
856
- value: function any(values) {
857
- const runtime = currentFunctionRuntime();
858
- if (runtime?.trackMixedPromiseRace == null) {
859
- return Reflect.apply(originalAny, this, [values]);
860
- }
861
- const durableContenders = [];
862
- let ordinaryContenderCount = 0;
863
- const observation = promiseAggregateObservation("any");
864
- const trackedValues = {
865
- *[Symbol.iterator]() {
866
- for (const value of values) {
867
- yield value;
868
- }
869
- }
870
- };
871
- const observedConstructor = observedPromiseConstructor(
872
- this,
873
- (normalized) => {
874
- if (isDurableThenable(normalized)) {
875
- durableContenders.push(normalized);
876
- return normalized;
877
- }
878
- ordinaryContenderCount += 1;
879
- return observePromiseAggregateContender(normalized, observation);
880
- }
881
- );
882
- const construct = () => Reflect.apply(
883
- originalAny,
884
- observedConstructor,
885
- [trackedValues]
886
- );
887
- const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
888
- observation.finishIteration(ordinaryContenderCount);
889
- if (durableContenders.length > 0) {
890
- trackDurableAggregate(
891
- runtime,
892
- result,
893
- durableContenders,
894
- !isInternalPromiseAggregate()
895
- );
896
- }
897
- if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
898
- if (observation.allOrdinaryValuesObservable) {
899
- runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
900
- } else {
901
- runtime.trackMixedPromiseRace(result);
902
- }
903
- }
904
- return result;
905
- }
906
- });
907
- Object.defineProperty(Promise, "all", {
908
- ...Object.getOwnPropertyDescriptor(Promise, "all"),
909
- value: function all(values) {
910
- const runtime = currentFunctionRuntime();
911
- if (runtime?.trackMixedPromiseRace == null) {
912
- return Reflect.apply(originalAll, this, [values]);
913
- }
914
- const durableContenders = [];
915
- let ordinaryContenderCount = 0;
916
- const observation = promiseAggregateObservation("all");
917
- const trackedValues = {
918
- *[Symbol.iterator]() {
919
- for (const value of values) {
920
- yield value;
921
- }
922
- }
923
- };
924
- const observedConstructor = observedPromiseConstructor(
925
- this,
926
- (normalized) => {
927
- if (isDurableThenable(normalized)) {
928
- durableContenders.push(normalized);
929
- return normalized;
930
- }
931
- ordinaryContenderCount += 1;
932
- return observePromiseAggregateContender(normalized, observation);
933
- }
934
- );
935
- const construct = () => Reflect.apply(
936
- originalAll,
937
- observedConstructor,
938
- [trackedValues]
939
- );
940
- const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
941
- observation.finishIteration(ordinaryContenderCount);
942
- if (durableContenders.length > 0) {
943
- trackDurableAggregate(
944
- runtime,
945
- result,
946
- durableContenders,
947
- !isInternalPromiseAggregate()
948
- );
949
- }
950
- if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
951
- if (observation.allOrdinaryValuesObservable) {
952
- runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
953
- } else {
954
- runtime.trackMixedPromiseRace(result);
955
- }
956
- }
957
- return result;
958
- }
959
- });
960
- Object.defineProperty(Promise, "allSettled", {
961
- ...Object.getOwnPropertyDescriptor(Promise, "allSettled"),
962
- value: function allSettled(values) {
963
- const runtime = currentFunctionRuntime();
964
- if (runtime?.trackMixedPromiseRace == null) {
965
- return Reflect.apply(
966
- originalAllSettled,
967
- this,
968
- [values]
969
- );
970
- }
971
- const durableContenders = [];
972
- let ordinaryContenderCount = 0;
973
- const observation = promiseAggregateObservation("all_settled");
974
- const trackedValues = {
975
- *[Symbol.iterator]() {
976
- for (const value of values) yield value;
977
- }
978
- };
979
- const observedConstructor = observedPromiseConstructor(
980
- this,
981
- (normalized) => {
982
- if (isDurableThenable(normalized)) {
983
- durableContenders.push(normalized);
984
- return normalized;
985
- }
986
- ordinaryContenderCount += 1;
987
- return observePromiseAggregateContender(normalized, observation);
988
- }
989
- );
990
- const construct = () => Reflect.apply(
991
- originalAllSettled,
992
- observedConstructor,
993
- [trackedValues]
994
- );
995
- const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
996
- observation.finishIteration(ordinaryContenderCount);
997
- if (durableContenders.length > 0) {
998
- trackDurableAggregate(
999
- runtime,
1000
- result,
1001
- durableContenders,
1002
- !isInternalPromiseAggregate()
1003
- );
1004
- }
1005
- if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
1006
- if (observation.allOrdinaryValuesObservable) {
1007
- runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
1008
- } else {
1009
- runtime.trackMixedPromiseRace(result);
1010
- }
1011
- }
1012
- return result;
1013
- }
1014
- });
1015
- target[PROMISE_INSTRUMENTED_KEY] = true;
1016
- }
1017
- function validateArguments(definition, supplied) {
1018
- const args = [...supplied];
1019
- if (args.length > definition.parameters.length) {
1020
- throw new SDKUsageError(`${definition.name} expected at most ${definition.parameters.length} arguments, received ${args.length}`);
1021
- }
1022
- definition.parameters.forEach((parameter, index) => {
1023
- if (index >= args.length || args[index] === void 0) {
1024
- if (parameter.hasDefault) args[index] = parameter.defaultValue;
1025
- else if (parameter.required) throw new SDKUsageError(`${definition.name} is missing argument '${parameter.name}'`);
1026
- return;
1027
- }
1028
- validateWithSchema(parameter.schema, args[index], `${definition.name}.${parameter.name}`);
1029
- });
1030
- return args;
1031
- }
1032
- async function executeHandlerResult(definition, suppliedArgs) {
1033
- const args = validateArguments(definition, suppliedArgs);
1034
- const result = definition.handler(...args);
1035
- if (result == null || typeof result.then !== "function") {
1036
- throw new SDKUsageError(`Tensorlake handler '${definition.name}' must be async and return a Promise`);
1037
- }
1038
- const output = await result;
1039
- if (isTailCall(output)) {
1040
- return output;
1041
- }
1042
- validateWithSchema(definition.returns, output, `${definition.name} return value`);
1043
- return output;
1044
- }
1045
- function isTailCall(value) {
1046
- if (typeof value !== "object" || value == null) return false;
1047
- const candidate = value;
1048
- return candidate[TAIL_CALL_BRAND] === true && candidate.kind === "tensorlake-tail-call" && typeof candidate.definition === "object" && candidate.definition != null && Array.isArray(candidate.args);
1049
- }
1050
-
1051
- // src/applications/headers.ts
1052
- var HEADERS_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.headers.v1");
1053
- var Headers2 = class _Headers {
1054
- [HEADERS_BRAND] = true;
1055
- #items;
1056
- #lookup;
1057
- static [Symbol.hasInstance](value) {
1058
- if (this !== _Headers) {
1059
- return Function.prototype[Symbol.hasInstance].call(this, value);
1060
- }
1061
- return isHeaders(value);
1062
- }
1063
- constructor(headers = []) {
1064
- const entries = headers instanceof _Headers ? [...headers] : Symbol.iterator in Object(headers) ? [...headers] : Object.entries(headers);
1065
- const lookup = /* @__PURE__ */ new Map();
1066
- this.#items = Object.freeze(entries.map(([name, value]) => {
1067
- if (typeof name !== "string" || typeof value !== "string") {
1068
- throw new TypeError("Headers require string names and values");
1069
- }
1070
- const item = Object.freeze([name, value]);
1071
- const key = name.toLowerCase();
1072
- lookup.set(key, [...lookup.get(key) ?? [], value]);
1073
- return item;
1074
- }));
1075
- this.#lookup = lookup;
1076
- Object.freeze(this);
1077
- }
1078
- get(name) {
1079
- return this.#lookup.get(name.toLowerCase())?.at(-1);
1080
- }
1081
- getAll(name) {
1082
- return Object.freeze([...this.#lookup.get(name.toLowerCase()) ?? []]);
1083
- }
1084
- has(name) {
1085
- return this.#lookup.has(name.toLowerCase());
1086
- }
1087
- entries() {
1088
- return this.#items[Symbol.iterator]();
1089
- }
1090
- [Symbol.iterator]() {
1091
- return this.entries();
1092
- }
1093
- };
1094
- function isHeaders(value) {
1095
- if (typeof value !== "object" || value == null) return false;
1096
- const candidate = value;
1097
- return candidate[HEADERS_BRAND] === true && typeof candidate.get === "function" && typeof candidate.getAll === "function" && typeof candidate.has === "function" && typeof candidate[Symbol.iterator] === "function";
1098
- }
1099
-
1100
- // src/applications/context.ts
1101
- import { AsyncLocalStorage as AsyncLocalStorage2 } from "async_hooks";
1102
-
1103
- // src/applications/serialization.ts
1104
- import { createHash } from "crypto";
1105
- var textEncoder = new TextEncoder();
1106
- var textDecoder = new TextDecoder("utf-8", { fatal: true });
1107
- function assertJSONValue(value, path3, seen) {
1108
- if (value === null || typeof value === "string" || typeof value === "boolean") return;
1109
- if (typeof value === "number") {
1110
- if (!Number.isFinite(value)) throw new SerializationError(`${path3} contains a non-finite number`);
1111
- return;
1112
- }
1113
- if (typeof value !== "object") {
1114
- throw new SerializationError(`${path3} contains unsupported ${typeof value}`);
1115
- }
1116
- if (isFile(value)) {
1117
- throw new SerializationError(`${path3} contains a nested File; File is only supported as a direct value`);
1118
- }
1119
- if (isHttpBody(value)) {
1120
- throw new SerializationError(
1121
- `${path3} contains a nested HttpBody; HttpBody is only supported as a direct value`
1122
- );
1123
- }
1124
- if (seen.has(value)) throw new SerializationError(`${path3} contains a cycle`);
1125
- if (Object.getPrototypeOf(value) !== Object.prototype && !Array.isArray(value)) {
1126
- throw new SerializationError(`${path3} contains unsupported ${value.constructor?.name ?? "object"}`);
1127
- }
1128
- seen.add(value);
1129
- if (Array.isArray(value)) {
1130
- for (let index = 0; index < value.length; index += 1) {
1131
- if (!Object.prototype.hasOwnProperty.call(value, index)) {
1132
- throw new SerializationError(`${path3} contains a sparse array entry at index ${index}`);
1133
- }
1134
- assertJSONValue(value[index], `${path3}[${index}]`, seen);
1135
- }
1136
- } else {
1137
- for (const [key, item] of Object.entries(value)) {
1138
- assertJSONValue(item, `${path3}.${key}`, seen);
1139
- }
1140
- }
1141
- seen.delete(value);
1142
- }
1143
- function serializeValue(value) {
1144
- if (isFile(value)) {
1145
- return { data: value.content, contentType: value.contentType, encoding: "raw" };
1146
- }
1147
- if (isHttpBody(value)) {
1148
- return {
1149
- data: value.content,
1150
- contentType: value.contentType ?? "",
1151
- encoding: "raw"
1152
- };
1153
- }
1154
- try {
1155
- assertJSONValue(value, "value", /* @__PURE__ */ new Set());
1156
- return {
1157
- data: textEncoder.encode(JSON.stringify(value)),
1158
- contentType: "application/json; charset=UTF-8",
1159
- encoding: "json"
1160
- };
1161
- } catch (error) {
1162
- if (error instanceof SerializationError) throw error;
1163
- throw new SerializationError("Failed to serialize JSON value", { cause: error });
1164
- }
1165
- }
1166
- function deserializeJSON(data) {
1167
- try {
1168
- return JSON.parse(textDecoder.decode(data));
1169
- } catch (error) {
1170
- throw new DeserializationError("Failed to deserialize JSON value", { cause: error });
1171
- }
1172
- }
1173
-
1174
- // src/applications/context.ts
1175
- function validateRequestStateKey(key) {
1176
- if (typeof key !== "string") {
1177
- throw new SDKUsageError(`State key must be a string, got: ${String(key)}`);
1178
- }
1179
- }
1180
- function serializeRequestStateValue(value) {
1181
- const serialized = serializeValue(value);
1182
- if (serialized.encoding !== "json") {
1183
- throw new SDKUsageError("Request state values must be JSON values");
1184
- }
1185
- return serialized.data;
1186
- }
1187
- function validateCounterMetric(name, value) {
1188
- if (typeof name !== "string") {
1189
- throw new SDKUsageError(`Counter name must be a string, got: ${String(name)}`);
1190
- }
1191
- if (typeof value !== "number" || !Number.isInteger(value)) {
1192
- throw new SDKUsageError(`Counter value must be an int, got: ${String(value)}`);
1193
- }
1194
- if (!Number.isSafeInteger(value)) {
1195
- throw new SDKUsageError(`Counter value must be a safe int, got: ${String(value)}`);
1196
- }
1197
- }
1198
- function validateTimerMetric(name, value) {
1199
- if (typeof name !== "string") {
1200
- throw new SDKUsageError(`Timer name must be a string, got: ${String(name)}`);
1201
- }
1202
- if (typeof value !== "number" || !Number.isFinite(value)) {
1203
- throw new SDKUsageError(`Timer value must be a finite number, got: ${String(value)}`);
1204
- }
1205
- }
1206
- function isPlainRecord(value) {
1207
- if (value == null || typeof value !== "object" || Array.isArray(value)) return false;
1208
- const prototype = Object.getPrototypeOf(value);
1209
- return prototype === Object.prototype || prototype === null;
1210
- }
1211
- function validateProgressUpdate(current, total, options) {
1212
- if (!Number.isFinite(current) || !Number.isFinite(total) || total < 0 || current < 0) {
1213
- throw new SDKUsageError("Progress current and total must be non-negative finite numbers");
1214
- }
1215
- if (options != null && !isPlainRecord(options)) {
1216
- throw new SDKUsageError("Progress options must be an object");
1217
- }
1218
- if (options?.message != null && typeof options.message !== "string") {
1219
- throw new SDKUsageError("Progress message must be a string");
1220
- }
1221
- if (options?.attributes != null) {
1222
- if (!isPlainRecord(options.attributes)) {
1223
- throw new SDKUsageError("Progress attributes must be an object of string key/value pairs");
1224
- }
1225
- for (const key of Reflect.ownKeys(options.attributes)) {
1226
- if (typeof key !== "string") {
1227
- throw new SDKUsageError("Progress attributes must contain only string keys and values");
1228
- }
1229
- const value = options.attributes[key];
1230
- if (typeof value !== "string") {
1231
- throw new SDKUsageError("Progress attributes must contain only string keys and values");
1232
- }
1233
- }
1234
- }
1235
- }
1236
- var REQUEST_CONTEXT_STORAGE_KEY = /* @__PURE__ */ Symbol.for(
1237
- "tensorlake.applications.request-context-storage.v1"
1238
- );
1239
- function requestContextStorage() {
1240
- const target = globalThis;
1241
- target[REQUEST_CONTEXT_STORAGE_KEY] ??= new AsyncLocalStorage2();
1242
- return target[REQUEST_CONTEXT_STORAGE_KEY];
1243
- }
1244
- function runWithRequestContext(context, callback) {
1245
- return requestContextStorage().run(context, callback);
1246
- }
1247
- async function waitWithAbortSignal(value, signal) {
1248
- if (signal.aborted) throw signal.reason;
1249
- let onAbort;
1250
- try {
1251
- return await Promise.race([
1252
- value,
1253
- new Promise((_resolve, reject) => {
1254
- onAbort = () => reject(signal.reason);
1255
- signal.addEventListener("abort", onAbort, { once: true });
1256
- })
1257
- ]);
1258
- } finally {
1259
- if (onAbort != null) signal.removeEventListener("abort", onAbort);
1260
- }
1261
- }
1262
-
1263
- // src/function-executor/async-queue.ts
1264
- function deferred() {
1265
- let resolve;
1266
- let reject;
1267
- const promise = new Promise((ok, fail) => {
1268
- resolve = ok;
1269
- reject = fail;
1270
- });
1271
- return { promise, resolve, reject };
1272
- }
1273
-
1274
- // src/function-executor/blob.ts
1275
- import { createHash as createHash2 } from "crypto";
1276
- import { open } from "fs/promises";
1277
- import { fileURLToPath } from "url";
1278
- function enumIs(value, name, numberValue) {
1279
- return value === name || value === numberValue;
1280
- }
1281
- function concatenate(parts) {
1282
- const result = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
1283
- let offset = 0;
1284
- for (const part of parts) {
1285
- result.set(part, offset);
1286
- offset += part.byteLength;
1287
- }
1288
- return result;
1289
- }
1290
- function networkURI(uri) {
1291
- if (uri.startsWith("s3://")) return `https://${uri.slice("s3://".length)}`;
1292
- if (uri.startsWith("gs://")) return `https://${uri.slice("gs://".length)}`;
1293
- return uri;
1294
- }
1295
- var BlobHTTPError = class extends Error {
1296
- constructor(status2) {
1297
- super(`BLOB operation failed with HTTP ${status2}`);
1298
- this.status = status2;
1299
- }
1300
- };
1301
- function uriKind(uri) {
1302
- if (uri.startsWith("file://")) return "file";
1303
- if (uri.startsWith("s3://")) return "s3";
1304
- if (uri.startsWith("gs://")) return "gcs";
1305
- try {
1306
- return new URL(uri).protocol.replace(":", "") || "unknown";
1307
- } catch {
1308
- return "unknown";
1309
- }
1310
- }
1311
- function throwIfAborted(signal) {
1312
- signal?.throwIfAborted();
1313
- }
1314
- async function abortableDelay(delayMs, signal) {
1315
- throwIfAborted(signal);
1316
- if (signal == null) {
1317
- await new Promise((resolve) => setTimeout(resolve, delayMs));
1318
- return;
1319
- }
1320
- await new Promise((resolve, reject) => {
1321
- const timer = setTimeout(() => {
1322
- signal.removeEventListener("abort", onAbort);
1323
- resolve();
1324
- }, delayMs);
1325
- const onAbort = () => {
1326
- clearTimeout(timer);
1327
- reject(signal.reason);
1328
- };
1329
- signal.addEventListener("abort", onAbort, { once: true });
1330
- });
1331
- }
1332
- async function fetchWithRetries(uri, init, log, fields = {}, signal) {
1333
- let lastError;
1334
- for (let attempt = 0; attempt < 4; attempt += 1) {
1335
- throwIfAborted(signal);
1336
- const startedAt = Date.now();
1337
- log?.("HTTP attempt starting", {
1338
- ...fields,
1339
- attempt: attempt + 1,
1340
- method: init?.method ?? "GET",
1341
- uri_kind: uriKind(uri),
1342
- has_range: new Headers(init?.headers).has("range")
1343
- });
1344
- try {
1345
- const response = await fetch(networkURI(uri), {
1346
- ...init,
1347
- signal
1348
- });
1349
- log?.("HTTP attempt completed", {
1350
- ...fields,
1351
- attempt: attempt + 1,
1352
- method: init?.method ?? "GET",
1353
- status_code: response.status,
1354
- duration_ms: Date.now() - startedAt
1355
- });
1356
- throwIfAborted(signal);
1357
- if (response.ok) return response;
1358
- if ([400, 403, 404].includes(response.status) || attempt === 3) {
1359
- throw new BlobHTTPError(response.status);
1360
- }
1361
- lastError = new BlobHTTPError(response.status);
1362
- } catch (error) {
1363
- log?.("HTTP attempt failed", {
1364
- ...fields,
1365
- attempt: attempt + 1,
1366
- method: init?.method ?? "GET",
1367
- duration_ms: Date.now() - startedAt,
1368
- error: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
1369
- });
1370
- throwIfAborted(signal);
1371
- if (error instanceof BlobHTTPError && [400, 403, 404].includes(error.status)) throw error;
1372
- lastError = error;
1373
- if (attempt === 3) break;
1374
- }
1375
- const retryDelayMs = 100 * 2 ** attempt;
1376
- log?.("HTTP attempt scheduled for retry", {
1377
- ...fields,
1378
- next_attempt: attempt + 2,
1379
- retry_delay_ms: retryDelayMs
1380
- });
1381
- await abortableDelay(retryDelayMs, signal);
1382
- }
1383
- throw new Error("BLOB operation failed after retries", { cause: lastError });
1384
- }
1385
- async function readChunkRange(chunk, index, logicalChunkOffset, offsetWithinChunk, size, sharedURI, log, signal) {
1386
- throwIfAborted(signal);
1387
- if (!chunk.uri) throw new Error("BLOB chunk has no URI");
1388
- const physicalOffset = sharedURI ? logicalChunkOffset + offsetWithinChunk : offsetWithinChunk;
1389
- const fields = {
1390
- chunk_index: index,
1391
- chunk_offset: logicalChunkOffset,
1392
- read_offset: physicalOffset,
1393
- read_bytes: size,
1394
- uri_kind: uriKind(chunk.uri),
1395
- shared_uri: sharedURI
1396
- };
1397
- const startedAt = Date.now();
1398
- log?.("download chunk starting", fields);
1399
- if (chunk.uri.startsWith("file://")) {
1400
- const file = await open(fileURLToPath(chunk.uri), "r");
1401
- try {
1402
- throwIfAborted(signal);
1403
- const data2 = new Uint8Array(size);
1404
- const { bytesRead } = await file.read(data2, 0, size, physicalOffset);
1405
- throwIfAborted(signal);
1406
- if (bytesRead !== size) {
1407
- throw new Error(`BLOB chunk ended after ${bytesRead} bytes; expected ${size}`);
1408
- }
1409
- log?.("download chunk completed", {
1410
- ...fields,
1411
- downloaded_bytes: data2.byteLength,
1412
- duration_ms: Date.now() - startedAt
1413
- });
1414
- return data2;
1415
- } finally {
1416
- await file.close();
1417
- }
1418
- }
1419
- const headers = { range: `bytes=${physicalOffset}-${physicalOffset + size - 1}` };
1420
- const response = await fetchWithRetries(chunk.uri, { headers }, log, fields, signal);
1421
- let data = new Uint8Array(await response.arrayBuffer());
1422
- throwIfAborted(signal);
1423
- if (response.status === 200 && (physicalOffset !== 0 || data.byteLength !== size)) {
1424
- data = data.subarray(physicalOffset, physicalOffset + size);
1425
- }
1426
- if (data.byteLength !== size) {
1427
- throw new Error(`BLOB chunk returned ${data.byteLength} bytes; expected ${size}`);
1428
- }
1429
- log?.("download chunk completed", {
1430
- ...fields,
1431
- status_code: response.status,
1432
- downloaded_bytes: data.byteLength,
1433
- duration_ms: Date.now() - startedAt
1434
- });
1435
- return data;
1436
- }
1437
- async function downloadBlobRange(blob, offset, size, log, signal) {
1438
- throwIfAborted(signal);
1439
- if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(size) || size < 0) {
1440
- throw new Error("BLOB range offset and size must be non-negative safe integers");
1441
- }
1442
- const chunks = blob.chunks ?? [];
1443
- const totalSize = chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0);
1444
- if (offset + size > totalSize) {
1445
- throw new Error(`BLOB range ${offset}..${offset + size} exceeds declared size ${totalSize}`);
1446
- }
1447
- if (size === 0) return new Uint8Array();
1448
- const uriCounts = /* @__PURE__ */ new Map();
1449
- for (const chunk of chunks) {
1450
- if (chunk.uri != null) uriCounts.set(chunk.uri, (uriCounts.get(chunk.uri) ?? 0) + 1);
1451
- }
1452
- const rangeEnd = offset + size;
1453
- let chunkOffset = 0;
1454
- const reads = [];
1455
- for (const [index, chunk] of chunks.entries()) {
1456
- const chunkSize = chunk.size ?? 0;
1457
- const chunkEnd = chunkOffset + chunkSize;
1458
- const overlapStart = Math.max(offset, chunkOffset);
1459
- const overlapEnd = Math.min(rangeEnd, chunkEnd);
1460
- if (overlapStart < overlapEnd) {
1461
- reads.push(readChunkRange(
1462
- chunk,
1463
- index,
1464
- chunkOffset,
1465
- overlapStart - chunkOffset,
1466
- overlapEnd - overlapStart,
1467
- (uriCounts.get(chunk.uri ?? "") ?? 0) > 1,
1468
- log,
1469
- signal
1470
- ));
1471
- }
1472
- chunkOffset = chunkEnd;
1473
- }
1474
- const data = concatenate(await Promise.all(reads));
1475
- throwIfAborted(signal);
1476
- if (data.byteLength !== size) throw new Error("BLOB does not contain the complete requested range");
1477
- log?.("download blob completed", {
1478
- blob_id: blob.id,
1479
- chunk_count: chunks.length,
1480
- downloaded_bytes: data.byteLength,
1481
- range_offset: offset,
1482
- range_size: size
1483
- });
1484
- return data;
1485
- }
1486
- async function downloadBlob(blob, log, signal) {
1487
- const size = (blob.chunks ?? []).reduce((total, chunk) => total + (chunk.size ?? 0), 0);
1488
- return downloadBlobRange(blob, 0, size, log, signal);
1489
- }
1490
- async function downloadSerializedObject(object, blob, log, signal) {
1491
- throwIfAborted(signal);
1492
- const manifest = object.manifest;
1493
- if (manifest?.size == null || manifest.metadataSize == null) {
1494
- throw new Error("Serialized object manifest is missing size or metadata_size");
1495
- }
1496
- if (!Number.isSafeInteger(manifest.size) || manifest.size < 0) {
1497
- throw new Error("Serialized object size must be a non-negative safe integer");
1498
- }
1499
- if (!Number.isSafeInteger(manifest.metadataSize) || manifest.metadataSize < 0) {
1500
- throw new Error("Serialized object metadata_size must be a non-negative safe integer");
1501
- }
1502
- if (manifest.metadataSize > manifest.size) {
1503
- throw new Error("Serialized object metadata_size cannot exceed size");
1504
- }
1505
- if (typeof manifest.sha256Hash !== "string" || manifest.sha256Hash.length === 0) {
1506
- throw new Error("Serialized object manifest is missing sha256_hash");
1507
- }
1508
- const start = object.offset ?? 0;
1509
- const bytes = await downloadBlobRange(blob, start, manifest.size, log, signal);
1510
- throwIfAborted(signal);
1511
- const hash = createHash2("sha256").update(bytes).digest("hex");
1512
- if (hash !== manifest.sha256Hash) {
1513
- throw new Error(`Serialized object hash mismatch: expected ${manifest.sha256Hash}, got ${hash}`);
1514
- }
1515
- return {
1516
- metadata: bytes.subarray(0, manifest.metadataSize),
1517
- data: bytes.subarray(manifest.metadataSize),
1518
- contentType: manifest.contentType,
1519
- encoding: manifest.encoding
1520
- };
1521
- }
1522
- function deserializeValueFromProtocol(value) {
1523
- if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_RAW", 5)) {
1524
- return new File(value.data, value.contentType ?? "application/octet-stream");
1525
- }
1526
- if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_UTF8_TEXT", 2)) {
1527
- return new TextDecoder("utf-8", { fatal: true }).decode(value.data);
1528
- }
1529
- if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_UTF8_JSON", 1) || value.contentType?.toLowerCase().includes("json")) {
1530
- return deserializeJSON(value.data);
1531
- }
1532
- throw new Error(`Unsupported serialized object encoding '${String(value.encoding)}'`);
1533
- }
1534
- function prepareSerializedObject(value, offset = 0, sourceFunctionCallId) {
1535
- const serialized = serializeValue(value);
1536
- const metadata = new TextEncoder().encode(JSON.stringify({
1537
- format: "tensorlake.typescript.value.v1"
1538
- }));
1539
- const bytes = concatenate([metadata, serialized.data]);
1540
- return {
1541
- bytes,
1542
- object: {
1543
- offset,
1544
- manifest: {
1545
- encoding: serialized.encoding === "json" ? "SERIALIZED_OBJECT_ENCODING_UTF8_JSON" : "SERIALIZED_OBJECT_ENCODING_RAW",
1546
- encodingVersion: 0,
1547
- size: bytes.byteLength,
1548
- metadataSize: metadata.byteLength,
1549
- sha256Hash: createHash2("sha256").update(bytes).digest("hex"),
1550
- ...serialized.encoding === "raw" ? { contentType: serialized.contentType } : {},
1551
- sourceFunctionCallId
1552
- }
1553
- }
1554
- };
1555
- }
1556
- function prepareTextObject(value) {
1557
- const bytes = new TextEncoder().encode(value);
1558
- return {
1559
- bytes,
1560
- object: {
1561
- offset: 0,
1562
- manifest: {
1563
- encoding: "SERIALIZED_OBJECT_ENCODING_UTF8_TEXT",
1564
- encodingVersion: 0,
1565
- size: bytes.byteLength,
1566
- metadataSize: 0,
1567
- sha256Hash: createHash2("sha256").update(bytes).digest("hex")
1568
- }
1569
- }
1570
- };
1571
- }
1572
- async function uploadChunk(chunk, data, index, offset, log, signal) {
1573
- throwIfAborted(signal);
1574
- if (!chunk.uri) throw new Error("Writable BLOB chunk has no URI");
1575
- const fields = {
1576
- chunk_index: index,
1577
- chunk_offset: offset,
1578
- upload_bytes: data.byteLength,
1579
- chunk_capacity: chunk.size ?? 0,
1580
- uri_kind: uriKind(chunk.uri)
1581
- };
1582
- const startedAt = Date.now();
1583
- log?.("upload chunk starting", fields);
1584
- if (chunk.uri.startsWith("file://")) {
1585
- const filePath = fileURLToPath(chunk.uri);
1586
- let file;
1587
- try {
1588
- file = await open(filePath, "r+");
1589
- } catch (error) {
1590
- if (error.code !== "ENOENT") throw error;
1591
- file = await open(filePath, "w+");
1592
- }
1593
- try {
1594
- throwIfAborted(signal);
1595
- await file.write(data, 0, data.byteLength, offset);
1596
- throwIfAborted(signal);
1597
- } finally {
1598
- await file.close();
1599
- }
1600
- log?.("upload chunk completed", { ...fields, duration_ms: Date.now() - startedAt });
1601
- return { ...chunk, size: data.byteLength };
1602
- }
1603
- const response = await fetchWithRetries(chunk.uri, {
1604
- method: "PUT",
1605
- ...data.byteLength === 0 ? {} : {
1606
- body: Uint8Array.from(data).buffer
1607
- }
1608
- }, log, fields, signal);
1609
- throwIfAborted(signal);
1610
- const uploaded = {
1611
- ...chunk,
1612
- size: data.byteLength,
1613
- etag: response.headers.get("etag") ?? void 0
1614
- };
1615
- log?.("upload chunk completed", {
1616
- ...fields,
1617
- status_code: response.status,
1618
- has_etag: uploaded.etag != null,
1619
- duration_ms: Date.now() - startedAt
1620
- });
1621
- return uploaded;
1622
- }
1623
- async function uploadBlob(blob, data, log, signal) {
1624
- throwIfAborted(signal);
1625
- const chunks = blob.chunks ?? [];
1626
- const capacity = chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0);
1627
- if (data.byteLength > capacity) throw new Error(`BLOB capacity ${capacity} is smaller than ${data.byteLength}`);
1628
- let offset = 0;
1629
- const uploaded = [];
1630
- for (const [index, chunk] of chunks.entries()) {
1631
- const size = Math.min(chunk.size ?? 0, data.byteLength - offset);
1632
- if (size <= 0) break;
1633
- uploaded.push(await uploadChunk(
1634
- chunk,
1635
- data.subarray(offset, offset + size),
1636
- index,
1637
- offset,
1638
- log,
1639
- signal
1640
- ));
1641
- offset += size;
1642
- }
1643
- if (data.byteLength === 0 && chunks.length > 0) {
1644
- uploaded.push(await uploadChunk(chunks[0], data, 0, 0, log, signal));
1645
- }
1646
- throwIfAborted(signal);
1647
- if (offset !== data.byteLength) throw new Error(`Only uploaded ${offset} of ${data.byteLength} bytes`);
1648
- log?.("upload blob completed", {
1649
- blob_id: blob.id,
1650
- chunk_count: uploaded.length,
1651
- uploaded_bytes: data.byteLength
1652
- });
1653
- return { ...blob, chunks: uploaded };
1654
- }
1655
- function joinPrepared(values) {
1656
- return concatenate(values.map((value) => value.bytes));
1657
- }
1658
-
1659
- // src/function-executor/user-events.ts
1660
- import { randomUUID as randomUUID2 } from "crypto";
1661
- function printCloudEvent(data, options = {}) {
1662
- writeStructuredOutput("stdout", () => ({
1663
- specversion: "1.0",
1664
- id: randomUUID2(),
1665
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
1666
- type: options.type ?? "ai.tensorlake.event",
1667
- source: options.source ?? "/tensorlake/function_executor/events",
1668
- data,
1669
- ...options.message == null ? {} : { message: options.message }
1670
- }));
1671
- }
1672
-
1673
- // src/function-executor/allocation.ts
1674
- installPromiseInstrumentation();
1675
- var OK = 0;
1676
- var NOT_FOUND = 5;
1677
- var REPLAY_STRICT = /* @__PURE__ */ new Set(["REPLAY_MODE_STRICT", 1]);
1678
- var OUTCOME_SUCCESS = /* @__PURE__ */ new Set(["ALLOCATION_OUTCOME_CODE_SUCCESS", 1]);
1679
- var OUTCOME_FAILURE = /* @__PURE__ */ new Set(["ALLOCATION_OUTCOME_CODE_FAILURE", 2]);
1680
- var WATCHER_TIMED_OUT = /* @__PURE__ */ new Set(["FUNCTION_CALL_WATCHER_STATUS_TIMEDOUT", 2]);
1681
- var MAX_REDUCE_CALLS_PER_PLAN = 512;
1682
- var nativeStructuredClone = globalThis.structuredClone;
1683
- function statusError(status2, fallback) {
1684
- if (status2 == null || (status2.code ?? OK) === OK) return void 0;
1685
- return new FunctionError(status2.message || fallback);
1686
- }
1687
- function validatedEventLogClock(response, requestedAfterClock, createError) {
1688
- if (response.entries != null && !Array.isArray(response.entries)) {
1689
- throw createError("Allocation event log returned a non-array entries field");
1690
- }
1691
- const responseClock = Number(response.lastClock ?? requestedAfterClock);
1692
- if (!Number.isFinite(responseClock) || !Number.isSafeInteger(responseClock) || responseClock < 0) {
1693
- throw createError(
1694
- `Allocation event log returned invalid clock '${String(response.lastClock)}'`
1695
- );
1696
- }
1697
- if (responseClock < requestedAfterClock) {
1698
- throw createError(
1699
- `Allocation event log moved backwards from clock ${requestedAfterClock} to ${responseClock}`
1700
- );
1701
- }
1702
- if (response.hasMore && responseClock === requestedAfterClock) {
1703
- throw createError(
1704
- `Allocation event log reported more entries without advancing clock ${requestedAfterClock}`
1705
- );
1706
- }
1707
- if ((response.entries?.length ?? 0) > 0 && responseClock === requestedAfterClock) {
1708
- throw createError(
1709
- `Allocation event log returned entries without advancing clock ${requestedAfterClock}`
1710
- );
1711
- }
1712
- const eventFields = [
1713
- "functionCallCreated",
1714
- "functionCallWatcherCreated",
1715
- "functionCallWatcherResult"
1716
- ];
1717
- let previousEntryClock = requestedAfterClock;
1718
- for (const [index, entry] of (response.entries ?? []).entries()) {
1719
- if (entry == null || typeof entry !== "object") {
1720
- throw createError(`Allocation event log entry ${index} is not an object`);
1721
- }
1722
- const presentFields = eventFields.filter((field) => entry[field] != null);
1723
- if (presentFields.length !== 1) {
1724
- throw createError(
1725
- `Allocation event log entry ${index} has ${presentFields.length} recognized payloads; expected exactly one`
1726
- );
1727
- }
1728
- const entryClock = Number(entry.clock);
1729
- if (!Number.isSafeInteger(entryClock) || entryClock <= previousEntryClock || entryClock > responseClock) {
1730
- throw createError(
1731
- `Allocation event log entry ${index} has invalid clock '${String(entry.clock)}' after ${previousEntryClock} with page clock ${responseClock}`
1732
- );
1733
- }
1734
- previousEntryClock = entryClock;
1735
- }
1736
- return responseClock;
1737
- }
1738
- function durableHash(values) {
1739
- const hash = createHash3("sha256");
1740
- for (const value of values) {
1741
- const bytes = Buffer.from(value, "utf8");
1742
- const length = Buffer.alloc(8);
1743
- length.writeBigUInt64BE(BigInt(bytes.byteLength));
1744
- hash.update(length);
1745
- hash.update(bytes);
1746
- hash.update("|");
1747
- }
1748
- return hash.digest("hex");
1749
- }
1750
- function protocolTimestamp(date) {
1751
- const milliseconds = date.getTime();
1752
- return { seconds: Math.floor(milliseconds / 1e3), nanos: milliseconds % 1e3 * 1e6 };
1753
- }
1754
- function parseHTTPMessage(data) {
1755
- const marker = Buffer.from("\r\n\r\n");
1756
- const bytes = Buffer.from(data);
1757
- const split = bytes.indexOf(marker);
1758
- if (split < 0) throw new DeserializationError("Invalid message/http application input");
1759
- const lines = bytes.subarray(0, split).toString("utf8").split("\r\n");
1760
- lines.shift();
1761
- const headers = {};
1762
- for (const line of lines) {
1763
- const colon = line.indexOf(":");
1764
- if (colon > 0) headers[line.slice(0, colon).trim().toLowerCase()] = line.slice(colon + 1).trim();
1765
- }
1766
- return { headers, body: new Uint8Array(bytes.subarray(split + marker.byteLength)) };
1767
- }
1768
- async function deserializeApplicationArguments(definition, payload) {
1769
- let data = payload.data;
1770
- let suppliedContentType = payload.contentType;
1771
- let hasExplicitContentType = payload.contentType != null;
1772
- if (suppliedContentType?.toLowerCase().startsWith("message/http")) {
1773
- const parsed = parseHTTPMessage(data);
1774
- data = parsed.body;
1775
- const bodyContentType = parsed.headers["content-type"];
1776
- hasExplicitContentType = bodyContentType != null;
1777
- suppliedContentType = bodyContentType;
1778
- }
1779
- if (definition.parameters.length === 0) return { args: [] };
1780
- const firstParameter = definition.parameters[0];
1781
- if (definition.parameters.length === 1 && firstParameter?.schema._httpBody) {
1782
- return { args: [new HttpBody(data, suppliedContentType)] };
1783
- }
1784
- const contentType = suppliedContentType ?? "application/json";
1785
- if (contentType.toLowerCase().startsWith("multipart/form-data")) {
1786
- let form;
1787
- try {
1788
- form = await new Response(Uint8Array.from(data).buffer, {
1789
- headers: { "content-type": contentType }
1790
- }).formData();
1791
- } catch (error) {
1792
- throw new DeserializationError("Failed to deserialize multipart application input", { cause: error });
1793
- }
1794
- const args = [];
1795
- for (const [index, parameter] of definition.parameters.entries()) {
1796
- const part = form.get(parameter.name) ?? form.get(String(index));
1797
- if (part == null) {
1798
- args.push(void 0);
1799
- } else if (parameter.schema._file || parameter.schema._httpBody) {
1800
- const bytes = typeof part === "string" ? new TextEncoder().encode(part) : new Uint8Array(await part.arrayBuffer());
1801
- const partContentType = typeof part === "string" ? "application/octet-stream" : part.type || "application/octet-stream";
1802
- args.push(parameter.schema._httpBody ? new HttpBody(bytes, partContentType) : new File(bytes, partContentType));
1803
- } else {
1804
- try {
1805
- const bytes = typeof part === "string" ? new TextEncoder().encode(part) : new Uint8Array(await part.arrayBuffer());
1806
- args.push(deserializeJSON(bytes));
1807
- } catch (error) {
1808
- if (error instanceof DeserializationError) throw error;
1809
- throw new DeserializationError(
1810
- `Failed to deserialize multipart argument '${parameter.name}' as JSON`,
1811
- { cause: error }
1812
- );
1813
- }
1814
- }
1815
- }
1816
- return { args };
1817
- }
1818
- if (data.byteLength === 0 && (!firstParameter?.schema._file || !hasExplicitContentType)) {
1819
- return { args: [] };
1820
- }
1821
- if (firstParameter?.schema._file) return { args: [new File(data, contentType)] };
1822
- if (!contentType.toLowerCase().includes("json")) {
1823
- throw new DeserializationError(
1824
- `Expected application/json input for '${firstParameter?.name ?? "argument"}', got ${contentType}`
1825
- );
1826
- }
1827
- return { args: [deserializeJSON(data)] };
1828
- }
1829
- var PASSIVE_ASYNC_RESOURCE_TYPES = /* @__PURE__ */ new Set([
1830
- "PIPEWRAP",
1831
- "PIPESERVERWRAP",
1832
- "SIGNALWRAP",
1833
- "TCPSERVERWRAP",
1834
- "TCPWRAP",
1835
- "TLSWRAP",
1836
- "TTYWRAP",
1837
- "UDPWRAP"
1838
- ]);
1839
- var ReplayCausality = class {
1840
- constructor(onQuiescent) {
1841
- this.onQuiescent = onQuiescent;
1842
- this.hook = createHook({
1843
- init: (asyncId, type, triggerAsyncId, resource) => {
1844
- if (this.stopped) return;
1845
- const triggerPromise = this.resources.get(triggerAsyncId)?.promise?.deref();
1846
- const resumesDetachedDurable = type === "PROMISE" && triggerPromise != null && isDetachedDurableThenable(triggerPromise);
1847
- const children = this.children.get(triggerAsyncId) ?? /* @__PURE__ */ new Set();
1848
- children.add(asyncId);
1849
- this.children.set(triggerAsyncId, children);
1850
- if (type === "PROMISE" && typeof resource === "object" && resource != null) {
1851
- this.promiseIds.set(resource, asyncId);
1852
- }
1853
- const triggerOwners = new Set(this.owners.get(triggerAsyncId) ?? []);
1854
- this.resources.set(asyncId, {
1855
- type,
1856
- triggerAsyncId,
1857
- promise: type === "PROMISE" && typeof resource === "object" && resource != null ? new WeakRef(resource) : void 0,
1858
- pending: true
1859
- });
1860
- if (resumesDetachedDurable) {
1861
- this.nativeDurableAssimilations.add(asyncId);
1862
- this.promiseAggregateAssimilationScopes.at(-1)?.add(asyncId);
1863
- this.suspendDurableAssimilation(asyncId);
1864
- }
1865
- if (this.suppressOwnership) return;
1866
- const inherited = /* @__PURE__ */ new Set([
1867
- ...this.owners.get(executionAsyncId2()) ?? [],
1868
- ...triggerOwners
1869
- ]);
1870
- for (const token of inherited) this.addTokenToSubtree(asyncId, token);
1871
- },
1872
- promiseResolve: (asyncId) => {
1873
- if (this.stopped) return;
1874
- this.notifyPromiseSettlement(asyncId);
1875
- if (this.nativeDurableAssimilations.delete(asyncId)) {
1876
- this.resumeDurableAssimilation(asyncId);
1877
- }
1878
- if (this.explicitDurableAssimilations.delete(asyncId)) {
1879
- this.resumeDurableAssimilation(asyncId);
1880
- }
1881
- this.settleMixedPromiseProducerGroups(asyncId);
1882
- const currentOwners = this.owners.get(executionAsyncId2());
1883
- if (currentOwners != null) {
1884
- for (const token of currentOwners) this.addTokenToSubtree(asyncId, token);
1885
- }
1886
- this.setResourcePending(asyncId, false);
1887
- this.scheduleCheck();
1888
- },
1889
- destroy: (asyncId) => {
1890
- if (this.stopped) return;
1891
- this.notifyPromiseSettlement(asyncId);
1892
- if (this.nativeDurableAssimilations.delete(asyncId)) {
1893
- this.detachDurableAssimilation(asyncId);
1894
- }
1895
- if (this.explicitDurableAssimilations.delete(asyncId)) {
1896
- this.detachDurableAssimilation(asyncId);
1897
- }
1898
- this.settleMixedPromiseProducerGroups(asyncId);
1899
- this.setResourcePending(asyncId, false);
1900
- this.releaseDestroyedResource(asyncId);
1901
- this.scheduleCheck();
1902
- }
1903
- });
1904
- this.hook.enable();
1905
- }
1906
- hook;
1907
- children = /* @__PURE__ */ new Map();
1908
- resources = /* @__PURE__ */ new Map();
1909
- owners = /* @__PURE__ */ new Map();
1910
- provenance = /* @__PURE__ */ new Map();
1911
- tokenResources = /* @__PURE__ */ new Map();
1912
- tokenProducers = /* @__PURE__ */ new Map();
1913
- promiseIds = /* @__PURE__ */ new WeakMap();
1914
- excludedRoots = /* @__PURE__ */ new Set();
1915
- durableAssimilations = /* @__PURE__ */ new Map();
1916
- nativeDurableAssimilations = /* @__PURE__ */ new Set();
1917
- explicitDurableAssimilations = /* @__PURE__ */ new Set();
1918
- promiseAggregateAssimilationScopes = [];
1919
- promiseSettlementObservers = /* @__PURE__ */ new Map();
1920
- mixedPromiseProducerGroups = /* @__PURE__ */ new Set();
1921
- nextToken = 1;
1922
- checkScheduled = false;
1923
- checkHandle;
1924
- suppressOwnership = false;
1925
- stopped = false;
1926
- runRoot(callback) {
1927
- const resource = new AsyncResource("TensorlakeReplayCausality");
1928
- let result;
1929
- resource.runInAsyncScope(() => {
1930
- this.beginCurrentContinuation();
1931
- result = callback();
1932
- this.excludePromise(result);
1933
- });
1934
- resource.emitDestroy();
1935
- return result;
1936
- }
1937
- beginCurrentContinuation() {
1938
- if (this.stopped) return;
1939
- const asyncId = executionAsyncId2();
1940
- this.excludedRoots.delete(asyncId);
1941
- this.addTokenToSubtree(asyncId, this.nextToken);
1942
- this.nextToken += 1;
1943
- this.scheduleCheck();
1944
- }
1945
- excludePromise(promise) {
1946
- const asyncId = this.promiseIds.get(promise);
1947
- if (asyncId == null) return;
1948
- this.excludedRoots.add(asyncId);
1949
- this.removeOwnersFromSubtree(asyncId);
1950
- this.scheduleCheck();
1951
- }
1952
- suspendDurableAssimilation(asyncId) {
1953
- if (this.stopped) return false;
1954
- const count = this.durableAssimilations.get(asyncId) ?? 0;
1955
- this.durableAssimilations.set(asyncId, count + 1);
1956
- if (count === 0) {
1957
- this.excludedRoots.add(asyncId);
1958
- this.removeOwnersFromSubtree(asyncId);
1959
- }
1960
- this.scheduleCheck();
1961
- return true;
1962
- }
1963
- resumeDurableAssimilation(asyncId) {
1964
- if (this.stopped) return;
1965
- const count = this.durableAssimilations.get(asyncId);
1966
- if (count == null) return;
1967
- if (count > 1) {
1968
- this.durableAssimilations.set(asyncId, count - 1);
1969
- return;
1970
- }
1971
- this.durableAssimilations.delete(asyncId);
1972
- this.excludedRoots.delete(asyncId);
1973
- const currentOwners = this.owners.get(executionAsyncId2());
1974
- if (currentOwners != null) {
1975
- for (const token of currentOwners) this.addTokenToSubtree(asyncId, token);
1976
- }
1977
- }
1978
- detachDurableAssimilation(asyncId) {
1979
- if (this.stopped) return;
1980
- const count = this.durableAssimilations.get(asyncId);
1981
- if (count == null) return;
1982
- if (count > 1) {
1983
- this.durableAssimilations.set(asyncId, count - 1);
1984
- return;
1985
- }
1986
- this.durableAssimilations.delete(asyncId);
1987
- this.excludedRoots.delete(asyncId);
1988
- this.scheduleCheck();
1989
- }
1990
- hasPendingDurableAssimilation() {
1991
- return this.durableAssimilations.size > 0;
1992
- }
1993
- hasPendingProducer() {
1994
- if (this.mixedPromiseProducerGroups.size > 0) return true;
1995
- for (const producers of this.tokenProducers.values()) {
1996
- if (producers.size > 0) return true;
1997
- }
1998
- return false;
1999
- }
2000
- requestCheck() {
2001
- this.scheduleCheck();
2002
- }
2003
- trackMixedPromiseRace(promise, producers = [promise]) {
2004
- if (this.stopped) return;
2005
- const producerIds = /* @__PURE__ */ new Set();
2006
- for (const producer of producers) {
2007
- const asyncId = this.promiseIds.get(producer);
2008
- if (asyncId == null) continue;
2009
- if (this.resources.get(asyncId)?.pending !== true) {
2010
- this.scheduleCheck();
2011
- return;
2012
- }
2013
- producerIds.add(asyncId);
2014
- }
2015
- if (producerIds.size > 0) this.mixedPromiseProducerGroups.add(producerIds);
2016
- this.scheduleCheck();
2017
- }
2018
- trackPromiseSettlement(promise, onSettled) {
2019
- if (this.stopped) return false;
2020
- const asyncId = this.promiseIds.get(promise);
2021
- if (asyncId == null) return false;
2022
- if (this.resources.get(asyncId)?.pending !== true) {
2023
- onSettled();
2024
- return true;
2025
- }
2026
- const observers = this.promiseSettlementObservers.get(asyncId) ?? /* @__PURE__ */ new Set();
2027
- observers.add(onSettled);
2028
- this.promiseSettlementObservers.set(asyncId, observers);
2029
- return true;
2030
- }
2031
- trackDurablePromiseConsumption(promise) {
2032
- if (this.stopped) return;
2033
- const asyncId = this.promiseIds.get(promise);
2034
- if (asyncId == null || this.resources.get(asyncId)?.pending !== true || this.nativeDurableAssimilations.has(asyncId) || this.explicitDurableAssimilations.has(asyncId)) {
2035
- return;
2036
- }
2037
- this.explicitDurableAssimilations.add(asyncId);
2038
- this.promiseAggregateAssimilationScopes.at(-1)?.add(asyncId);
2039
- this.suspendDurableAssimilation(asyncId);
2040
- }
2041
- runPromiseAggregateConstruction(callback) {
2042
- const aggregateAssimilations = /* @__PURE__ */ new Set();
2043
- this.promiseAggregateAssimilationScopes.push(aggregateAssimilations);
2044
- try {
2045
- return callback();
2046
- } finally {
2047
- this.promiseAggregateAssimilationScopes.pop();
2048
- this.detachPromiseAggregateAssimilations(aggregateAssimilations);
2049
- }
2050
- }
2051
- runWithoutOwnership(callback) {
2052
- if (this.stopped) return callback();
2053
- const previous = this.suppressOwnership;
2054
- this.suppressOwnership = true;
2055
- try {
2056
- return callback();
2057
- } finally {
2058
- this.suppressOwnership = previous;
2059
- }
2060
- }
2061
- stop() {
2062
- if (this.stopped) return;
2063
- this.stopped = true;
2064
- this.hook.disable();
2065
- if (this.checkHandle != null) clearImmediate(this.checkHandle);
2066
- this.checkHandle = void 0;
2067
- this.children.clear();
2068
- this.resources.clear();
2069
- this.owners.clear();
2070
- this.provenance.clear();
2071
- this.tokenResources.clear();
2072
- this.tokenProducers.clear();
2073
- this.durableAssimilations.clear();
2074
- this.nativeDurableAssimilations.clear();
2075
- this.explicitDurableAssimilations.clear();
2076
- this.promiseAggregateAssimilationScopes.length = 0;
2077
- this.promiseSettlementObservers.clear();
2078
- this.mixedPromiseProducerGroups.clear();
2079
- }
2080
- notifyPromiseSettlement(asyncId) {
2081
- const observers = this.promiseSettlementObservers.get(asyncId);
2082
- if (observers == null) return;
2083
- this.promiseSettlementObservers.delete(asyncId);
2084
- for (const observer of observers) observer();
2085
- }
2086
- settleMixedPromiseProducerGroups(asyncId) {
2087
- for (const group of [...this.mixedPromiseProducerGroups]) {
2088
- if (group.has(asyncId)) this.mixedPromiseProducerGroups.delete(group);
2089
- }
2090
- }
2091
- detachPromiseAggregateAssimilations(assimilations) {
2092
- for (const asyncId of assimilations) {
2093
- if (this.nativeDurableAssimilations.delete(asyncId)) {
2094
- this.detachDurableAssimilation(asyncId);
2095
- }
2096
- if (this.explicitDurableAssimilations.delete(asyncId)) {
2097
- this.detachDurableAssimilation(asyncId);
2098
- }
2099
- }
2100
- assimilations.clear();
2101
- }
2102
- addTokenToSubtree(asyncId, token) {
2103
- if (this.excludedRoots.has(asyncId)) return;
2104
- const owners = this.owners.get(asyncId) ?? /* @__PURE__ */ new Set();
2105
- if (!owners.has(token)) {
2106
- owners.add(token);
2107
- this.owners.set(asyncId, owners);
2108
- const provenance = this.provenance.get(asyncId) ?? /* @__PURE__ */ new Set();
2109
- provenance.add(token);
2110
- this.provenance.set(asyncId, provenance);
2111
- const tokenResources = this.tokenResources.get(token) ?? /* @__PURE__ */ new Set();
2112
- tokenResources.add(asyncId);
2113
- this.tokenResources.set(token, tokenResources);
2114
- const resource = this.resources.get(asyncId);
2115
- if (resource?.pending && this.isProducer(resource, token)) {
2116
- const producers = this.tokenProducers.get(token) ?? /* @__PURE__ */ new Set();
2117
- producers.add(asyncId);
2118
- this.tokenProducers.set(token, producers);
2119
- }
2120
- }
2121
- for (const child of this.children.get(asyncId) ?? []) {
2122
- this.addTokenToSubtree(child, token);
2123
- }
2124
- }
2125
- isProducer(resource, token) {
2126
- if (resource.type === "PROMISE") {
2127
- return !(this.provenance.get(resource.triggerAsyncId)?.has(token) ?? false);
2128
- }
2129
- return !PASSIVE_ASYNC_RESOURCE_TYPES.has(resource.type);
2130
- }
2131
- setResourcePending(asyncId, pending) {
2132
- const resource = this.resources.get(asyncId);
2133
- if (resource == null || resource.pending === pending) return;
2134
- resource.pending = pending;
2135
- for (const token of this.owners.get(asyncId) ?? []) {
2136
- const producers = this.tokenProducers.get(token);
2137
- if (pending && this.isProducer(resource, token)) {
2138
- const next = producers ?? /* @__PURE__ */ new Set();
2139
- next.add(asyncId);
2140
- this.tokenProducers.set(token, next);
2141
- } else {
2142
- producers?.delete(asyncId);
2143
- }
2144
- }
2145
- }
2146
- removeOwnersFromSubtree(asyncId) {
2147
- this.removeResourceOwnership(asyncId);
2148
- for (const child of this.children.get(asyncId) ?? []) {
2149
- this.removeOwnersFromSubtree(child);
2150
- }
2151
- }
2152
- removeResourceOwnership(asyncId) {
2153
- const owners = this.owners.get(asyncId);
2154
- if (owners == null) return;
2155
- for (const token of owners) {
2156
- const resources = this.tokenResources.get(token);
2157
- resources?.delete(asyncId);
2158
- if (resources?.size === 0) this.tokenResources.delete(token);
2159
- const producers = this.tokenProducers.get(token);
2160
- producers?.delete(asyncId);
2161
- if (producers?.size === 0) this.tokenProducers.delete(token);
2162
- }
2163
- this.owners.delete(asyncId);
2164
- }
2165
- releaseDestroyedResource(asyncId) {
2166
- const resource = this.resources.get(asyncId);
2167
- if (resource == null) return;
2168
- this.resources.delete(asyncId);
2169
- this.excludedRoots.delete(asyncId);
2170
- this.removeResourceOwnership(asyncId);
2171
- this.provenance.delete(asyncId);
2172
- const siblings = this.children.get(resource.triggerAsyncId);
2173
- siblings?.delete(asyncId);
2174
- if (siblings?.size === 0) this.children.delete(resource.triggerAsyncId);
2175
- this.children.delete(asyncId);
2176
- }
2177
- scheduleCheck() {
2178
- if (this.stopped || this.checkScheduled) return;
2179
- this.checkScheduled = true;
2180
- const previous = this.suppressOwnership;
2181
- this.suppressOwnership = true;
2182
- try {
2183
- this.checkHandle = setImmediate(() => {
2184
- this.checkScheduled = false;
2185
- this.checkHandle = void 0;
2186
- this.onQuiescent();
2187
- });
2188
- } finally {
2189
- this.suppressOwnership = previous;
2190
- }
2191
- }
2192
- };
2193
- var ReplayAwarePromise = class extends Promise {
2194
- causality;
2195
- constructor(source, causality) {
2196
- let bridge;
2197
- super((resolve, reject) => {
2198
- bridge = source.then(resolve, reject);
2199
- });
2200
- this.causality = causality;
2201
- registerDurableThenable(this);
2202
- causality.excludePromise(source);
2203
- causality.excludePromise(this);
2204
- if (bridge != null) causality.excludePromise(bridge);
2205
- }
2206
- static get [Symbol.species]() {
2207
- return Promise;
2208
- }
2209
- then(onfulfilled, onrejected) {
2210
- const assimilationId = executionAsyncId2();
2211
- const assimilationSuspended = this.causality.suspendDurableAssimilation(assimilationId);
2212
- let assimilationDetached = false;
2213
- const aggregateAssimilation = registerDurableAssimilation(this, () => {
2214
- if (!assimilationSuspended || assimilationDetached) return;
2215
- assimilationDetached = true;
2216
- this.causality.detachDurableAssimilation(assimilationId);
2217
- });
2218
- const continuation = super.then(
2219
- (value) => {
2220
- this.causality.beginCurrentContinuation();
2221
- aggregateAssimilation.finish();
2222
- if (assimilationSuspended && !assimilationDetached) {
2223
- this.causality.resumeDurableAssimilation(assimilationId);
2224
- }
2225
- if (onfulfilled == null) return value;
2226
- return onfulfilled(value);
2227
- },
2228
- (reason) => {
2229
- this.causality.beginCurrentContinuation();
2230
- aggregateAssimilation.finish();
2231
- if (assimilationSuspended && !assimilationDetached) {
2232
- this.causality.resumeDurableAssimilation(assimilationId);
2233
- }
2234
- if (onrejected == null) throw reason;
2235
- return onrejected(reason);
2236
- }
2237
- );
2238
- aggregateAssimilation.link(continuation);
2239
- this.causality.excludePromise(continuation);
2240
- return continuation;
2241
- }
2242
- [DURABLE_PROMISE_INTERNAL_THEN](onfulfilled, onrejected) {
2243
- const continuation = super.then(onfulfilled, onrejected);
2244
- registerDurableThenable(continuation, this);
2245
- this.causality.excludePromise(continuation);
2246
- return continuation;
2247
- }
2248
- [DURABLE_PROMISE_OBSERVER](onfulfilled, onrejected) {
2249
- const observation = super.then(onfulfilled, onrejected);
2250
- this.causality.excludePromise(observation);
2251
- void observation.catch(() => void 0);
2252
- }
2253
- };
2254
- var ReplayHistory = class {
2255
- entries;
2256
- calls = [];
2257
- watchers = [];
2258
- availableResults = /* @__PURE__ */ new Map();
2259
- resultWaiters = /* @__PURE__ */ new Map();
2260
- endWaiters = [];
2261
- createdWatcherIds = /* @__PURE__ */ new Set();
2262
- releasedResultIds = /* @__PURE__ */ new Set();
2263
- expectedCallIds = /* @__PURE__ */ new Set();
2264
- expectedWatcherIds = /* @__PURE__ */ new Set();
2265
- causality;
2266
- // A replayed result may still be downloading before it can resume user code.
2267
- activeResultResolutions = 0;
2268
- callCursor = 0;
2269
- watcherCursor = 0;
2270
- prefixCursor = 0;
2271
- mismatch;
2272
- constructor(entries) {
2273
- this.causality = new ReplayCausality(() => this.failBlockedResultWithoutProducer());
2274
- this.entries = entries.map((entry) => {
2275
- if (entry.functionCallWatcherResult != null) {
2276
- return { kind: "result", event: entry.functionCallWatcherResult, consumed: false };
2277
- }
2278
- if (entry.functionCallCreated != null) {
2279
- const replayEntry = {
2280
- kind: "call",
2281
- event: entry.functionCallCreated,
2282
- consumed: false
2283
- };
2284
- this.calls.push(replayEntry);
2285
- return replayEntry;
2286
- }
2287
- if (entry.functionCallWatcherCreated != null) {
2288
- const replayEntry = {
2289
- kind: "watcher",
2290
- event: entry.functionCallWatcherCreated,
2291
- consumed: false
2292
- };
2293
- this.watchers.push(replayEntry);
2294
- return replayEntry;
2295
- }
2296
- return { kind: "unknown", event: entry, consumed: false };
2297
- });
2298
- }
2299
- async takeOrdered(kind, id) {
2300
- this.advancePrefix();
2301
- const entries = kind === "call" ? this.calls : this.watchers;
2302
- const cursor = kind === "call" ? this.callCursor : this.watcherCursor;
2303
- const expectedIds = kind === "call" ? this.expectedCallIds : this.expectedWatcherIds;
2304
- if (cursor >= entries.length) {
2305
- expectedIds.delete(id);
2306
- if (this.prefixCursor === this.entries.length) return void 0;
2307
- const blocked = this.entries[this.prefixCursor];
2308
- if (kind !== "call" || blocked.kind !== "watcher" || !this.expectedWatcherIds.has(String(blocked.event.functionCallId ?? ""))) {
2309
- this.fail(
2310
- `Unexpected ${kind} event for ${id} before replay ${blocked.kind} event ${String(blocked.event.functionCallId ?? "")}`
2311
- );
2312
- }
2313
- const waiter = deferred();
2314
- this.endWaiters.push(waiter);
2315
- await waiter.promise;
2316
- this.assertNoMismatch();
2317
- return void 0;
2318
- }
2319
- const entry = entries[cursor];
2320
- if (kind === "call") this.callCursor += 1;
2321
- else this.watcherCursor += 1;
2322
- if (entry.event.functionCallId !== id) {
2323
- this.fail(`Expected replay ${kind} event for ${id}`);
2324
- }
2325
- expectedIds.delete(id);
2326
- entry.consumed = true;
2327
- if (kind === "watcher") {
2328
- this.createdWatcherIds.add(id);
2329
- }
2330
- this.advancePrefix();
2331
- return entry.event;
2332
- }
2333
- async takeResult(id) {
2334
- this.advancePrefix();
2335
- const available = this.availableResults.get(id);
2336
- if (available != null && available.length > 0) {
2337
- const result2 = available.shift();
2338
- if (result2 != null) this.activeResultResolutions += 1;
2339
- return result2;
2340
- }
2341
- if (this.prefixCursor === this.entries.length) return void 0;
2342
- const blocked = this.entries[this.prefixCursor];
2343
- const blockedId = String(blocked.event.functionCallId ?? "");
2344
- const hasProducer = blocked.kind === "call" ? this.expectedCallIds.has(blockedId) : blocked.kind === "watcher" ? this.expectedWatcherIds.has(blockedId) : false;
2345
- const waiter = deferred();
2346
- const waiters = this.resultWaiters.get(id) ?? [];
2347
- waiters.push(waiter);
2348
- this.resultWaiters.set(id, waiters);
2349
- if (!hasProducer) this.causality.requestCheck();
2350
- const result = await waiter.promise;
2351
- if (result != null) this.activeResultResolutions += 1;
2352
- return result;
2353
- }
2354
- completeResultResolution() {
2355
- if (this.activeResultResolutions === 0) return;
2356
- this.activeResultResolutions -= 1;
2357
- this.causality.requestCheck();
2358
- }
2359
- runCausalHandler(callback) {
2360
- return this.causality.runRoot(callback);
2361
- }
2362
- wrapDurableResult(source) {
2363
- return new ReplayAwarePromise(source, this.causality);
2364
- }
2365
- trackMixedPromiseRace(promise, producers) {
2366
- this.causality.trackMixedPromiseRace(promise, producers);
2367
- }
2368
- trackPromiseSettlement(promise, onSettled) {
2369
- return this.causality.trackPromiseSettlement(promise, onSettled);
2370
- }
2371
- trackDurablePromiseConsumption(promise) {
2372
- this.causality.trackDurablePromiseConsumption(promise);
2373
- }
2374
- runPromiseAggregateConstruction(callback) {
2375
- return this.causality.runPromiseAggregateConstruction(callback);
2376
- }
2377
- runWithoutCausalOwnership(callback) {
2378
- return this.causality.runWithoutOwnership(callback);
2379
- }
2380
- dispose() {
2381
- this.causality.stop();
2382
- }
2383
- expectCall(id) {
2384
- this.expectedCallIds.add(id);
2385
- }
2386
- expectWatcher(id) {
2387
- this.expectedWatcherIds.add(id);
2388
- }
2389
- abandonCall(id) {
2390
- this.abandonOrdered("call", id);
2391
- }
2392
- abandonWatcher(id) {
2393
- this.abandonOrdered("watcher", id);
2394
- }
2395
- abandonOrdered(kind, id) {
2396
- const expectedIds = kind === "call" ? this.expectedCallIds : this.expectedWatcherIds;
2397
- expectedIds.delete(id);
2398
- const hasWaiters = this.endWaiters.length > 0 || [...this.resultWaiters.values()].some((waiters) => waiters.length > 0);
2399
- if (this.prefixCursor >= this.entries.length || !hasWaiters) return;
2400
- const blocked = this.entries[this.prefixCursor];
2401
- if (blocked.kind === kind && blocked.event.functionCallId === id) {
2402
- this.fail(`Replay ${kind} event for ${id} can no longer be created`);
2403
- }
2404
- }
2405
- failBlockedResultWithoutProducer() {
2406
- if (this.activeResultResolutions > 0) return;
2407
- if (!this.causality.hasPendingDurableAssimilation()) return;
2408
- if (this.causality.hasPendingProducer()) return;
2409
- if (this.prefixCursor >= this.entries.length) return;
2410
- if (![...this.resultWaiters.values()].some((waiters) => waiters.length > 0)) return;
2411
- const blocked = this.entries[this.prefixCursor];
2412
- const blockedId = String(blocked.event.functionCallId ?? "");
2413
- const hasProducer = blocked.kind === "call" ? this.expectedCallIds.has(blockedId) : blocked.kind === "watcher" ? this.expectedWatcherIds.has(blockedId) : false;
2414
- if (!hasProducer) {
2415
- this.recordMismatch(
2416
- `Replay result is blocked behind unreachable ${blocked.kind} event ${blockedId}`
2417
- );
2418
- }
2419
- }
2420
- hasAvailableResult(id) {
2421
- return (this.availableResults.get(id)?.length ?? 0) > 0;
2422
- }
2423
- assertNoMismatch() {
2424
- if (this.mismatch != null) throw this.mismatch;
2425
- }
2426
- assertConsumed() {
2427
- this.assertNoMismatch();
2428
- this.advancePrefix();
2429
- if (this.prefixCursor !== this.entries.length) {
2430
- throw new ReplayMismatchError("Function completed before its durable event history was consumed");
2431
- }
2432
- }
2433
- advancePrefix() {
2434
- this.assertNoMismatch();
2435
- while (this.prefixCursor < this.entries.length) {
2436
- const entry = this.entries[this.prefixCursor];
2437
- if (entry.kind === "unknown") {
2438
- this.fail("Replay history contains an unknown allocation event");
2439
- }
2440
- if (entry.kind === "call" || entry.kind === "watcher") {
2441
- if (!entry.consumed) return;
2442
- this.prefixCursor += 1;
2443
- continue;
2444
- }
2445
- entry.consumed = true;
2446
- this.prefixCursor += 1;
2447
- this.releaseResult(entry.event);
2448
- }
2449
- for (const waiter of this.endWaiters.splice(0)) waiter.resolve();
2450
- for (const waiters of this.resultWaiters.values()) {
2451
- for (const waiter of waiters) waiter.resolve(void 0);
2452
- }
2453
- this.resultWaiters.clear();
2454
- }
2455
- releaseResult(event) {
2456
- const id = String(event.functionCallId ?? "");
2457
- if (!this.createdWatcherIds.has(id)) {
2458
- this.fail(`Replay watcher result for ${id} appeared before its watcher was created`);
2459
- }
2460
- if (this.releasedResultIds.has(id)) {
2461
- this.fail(`Replay history contains duplicate watcher results for ${id}`);
2462
- }
2463
- this.releasedResultIds.add(id);
2464
- const waiter = this.resultWaiters.get(id)?.shift();
2465
- if (waiter != null) {
2466
- waiter.resolve(event);
2467
- if (this.resultWaiters.get(id)?.length === 0) this.resultWaiters.delete(id);
2468
- return;
2469
- }
2470
- const available = this.availableResults.get(id) ?? [];
2471
- available.push(event);
2472
- this.availableResults.set(id, available);
2473
- }
2474
- fail(message) {
2475
- throw this.recordMismatch(message);
2476
- }
2477
- recordMismatch(message) {
2478
- this.mismatch ??= new ReplayMismatchError(message);
2479
- this.causality.stop();
2480
- for (const waiter of this.endWaiters.splice(0)) waiter.reject(this.mismatch);
2481
- for (const waiters of this.resultWaiters.values()) {
2482
- for (const waiter of waiters) waiter.reject(this.mismatch);
2483
- }
2484
- this.resultWaiters.clear();
2485
- return this.mismatch;
2486
- }
2487
- };
2488
- var AllocationProtocolError = class extends Error {
2489
- constructor(message, options) {
2490
- super(message, options);
2491
- this.name = "AllocationProtocolError";
2492
- }
2493
- };
2494
- var AllocationRunner = class {
2495
- allocation;
2496
- functionRef;
2497
- definition;
2498
- applicationDefinition;
2499
- stateStreams = /* @__PURE__ */ new Set();
2500
- stateStreamFlows = /* @__PURE__ */ new Map();
2501
- eventReadStreams = /* @__PURE__ */ new Set();
2502
- outputBlobRequests = /* @__PURE__ */ new Map();
2503
- stateOperationRequests = /* @__PURE__ */ new Map();
2504
- executionBatches = [];
2505
- executionBatchWaiters = [];
2506
- liveBacklog = [];
2507
- liveWaiters = [];
2508
- controller = new AbortController();
2509
- currentRead;
2510
- replay;
2511
- protocolError;
2512
- durableCallEmissionTail = Promise.resolve();
2513
- durableWatcherEmissionTail = Promise.resolve();
2514
- lastEventClock = 0;
2515
- previousDurableId;
2516
- livePump;
2517
- completion;
2518
- started = false;
2519
- terminalBatchQueued = false;
2520
- terminalBatchObserved = false;
2521
- terminalBatchObservedWaiters = [];
2522
- finished = false;
2523
- state = { outputBlobRequests: [], requestStateOperations: [] };
2524
- startedAt = Date.now();
2525
- constructor(allocation, functionRef, definition, applicationDefinition = definition) {
2526
- this.allocation = allocation;
2527
- this.functionRef = functionRef;
2528
- this.definition = definition;
2529
- this.applicationDefinition = applicationDefinition;
2530
- this.previousDurableId = String(allocation.functionCallId);
2531
- this.updateStateHash();
2532
- this.log("debug", "allocation runner created", {
2533
- replay_mode: allocation.replayMode,
2534
- input_count: allocation.inputs?.args?.length ?? 0,
2535
- input_blob_count: allocation.inputs?.argBlobs?.length ?? 0
2536
- });
2537
- }
2538
- start() {
2539
- if (this.started) {
2540
- this.log("warn", "duplicate allocation runner start ignored");
2541
- return;
2542
- }
2543
- this.started = true;
2544
- this.log("info", "allocation runner scheduled");
2545
- this.completion = this.run().catch((error) => {
2546
- this.log("error", "allocation runner promise rejected", {}, error);
2547
- this.queueTerminalBatch({
2548
- outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
2549
- failureReason: "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR"
2550
- });
2551
- this.finish();
2552
- });
2553
- }
2554
- waitForCompletion() {
2555
- return this.completion ?? Promise.resolve();
2556
- }
2557
- async waitForTerminalBatchObserved(timeoutMs) {
2558
- if (this.terminalBatchObserved) return true;
2559
- return new Promise((resolve) => {
2560
- let settled = false;
2561
- const observed = () => {
2562
- if (settled) return;
2563
- settled = true;
2564
- clearTimeout(timer);
2565
- resolve(true);
2566
- };
2567
- const timer = setTimeout(() => {
2568
- if (settled) return;
2569
- settled = true;
2570
- const index = this.terminalBatchObservedWaiters.indexOf(observed);
2571
- if (index >= 0) this.terminalBatchObservedWaiters.splice(index, 1);
2572
- resolve(false);
2573
- }, Math.max(timeoutMs, 0));
2574
- this.terminalBatchObservedWaiters.push(observed);
2575
- if (this.terminalBatchObserved) observed();
2576
- });
2577
- }
2578
- cancel(reason = new FunctionError("Allocation was cancelled")) {
2579
- if (this.controller.signal.aborted) return;
2580
- this.log("info", "allocation cancellation requested", {
2581
- reason: reason instanceof Error ? reason.message : String(reason)
2582
- });
2583
- this.abortExecution(reason);
2584
- }
2585
- abortExecution(reason) {
2586
- if (!this.controller.signal.aborted) this.controller.abort(reason);
2587
- const rejection = this.controller.signal.reason ?? reason;
2588
- const outputBlobRequestCount = this.outputBlobRequests.size;
2589
- const stateOperationRequestCount = this.stateOperationRequests.size;
2590
- const liveWaiterCount = this.liveWaiters.length;
2591
- const liveBacklogCount = this.liveBacklog.length;
2592
- const hadCurrentRead = this.currentRead != null;
2593
- for (const request of this.outputBlobRequests.values()) request.reject(rejection);
2594
- for (const request of this.stateOperationRequests.values()) request.reject(rejection);
2595
- for (const waiter of this.liveWaiters.splice(0)) waiter.result.reject(rejection);
2596
- this.currentRead?.response.reject(rejection);
2597
- this.outputBlobRequests.clear();
2598
- this.stateOperationRequests.clear();
2599
- this.liveBacklog.splice(0);
2600
- this.currentRead = void 0;
2601
- this.state.outputBlobRequests = [];
2602
- this.state.requestStateOperations = [];
2603
- this.log("debug", "pending allocation protocol work settled", {
2604
- output_blob_requests: outputBlobRequestCount,
2605
- state_operation_requests: stateOperationRequestCount,
2606
- durable_event_waiters: liveWaiterCount,
2607
- durable_event_backlog_entries: liveBacklogCount,
2608
- event_log_read: hadCurrentRead
2609
- });
2610
- }
2611
- invoke(definition, args) {
2612
- this.log("debug", "durable function invocation requested", {
2613
- target_function: definition.name,
2614
- argument_count: args.length
2615
- });
2616
- const result = this.runChild(definition, args, 0);
2617
- return this.replay?.wrapDurableResult(result) ?? result;
2618
- }
2619
- runFuture(future) {
2620
- this.log("debug", "durable future execution requested", {
2621
- target_function: future.definition.name,
2622
- argument_count: future.args.length,
2623
- delay_seconds: future.delaySeconds
2624
- });
2625
- const result = this.runChild(future.definition, future.args, future.delaySeconds);
2626
- return this.replay?.wrapDurableResult(result) ?? result;
2627
- }
2628
- wrapFutureWait(promise) {
2629
- return this.replay?.wrapDurableResult(promise) ?? promise;
2630
- }
2631
- trackMixedPromiseRace(promise, producers) {
2632
- this.replay?.trackMixedPromiseRace(promise, producers);
2633
- }
2634
- trackPromiseSettlement(promise, onSettled) {
2635
- return this.replay?.trackPromiseSettlement(promise, onSettled) ?? false;
2636
- }
2637
- trackDurablePromiseConsumption(promise) {
2638
- this.replay?.trackDurablePromiseConsumption(promise);
2639
- }
2640
- runPromiseAggregateConstruction(callback) {
2641
- return this.replay?.runPromiseAggregateConstruction(callback) ?? callback();
2642
- }
2643
- reduce(definition, items, initial, hasInitial) {
2644
- const result = this.runReduce(definition, items, initial, hasInitial);
2645
- return this.replay?.wrapDurableResult(result) ?? result;
2646
- }
2647
- async runReduce(definition, items, initial, hasInitial) {
2648
- let reduceItems = items;
2649
- let reduceInitial = initial;
2650
- if (!hasInitial) {
2651
- if (items.length === 0) {
2652
- throw new SDKUsageError("reduce of empty iterable with no initial value");
2653
- }
2654
- [reduceInitial, ...reduceItems] = items;
2655
- }
2656
- if (reduceItems.length === 0) return reduceInitial;
2657
- const callTurn = this.reserveEmissionTurn("call");
2658
- const watcherTurn = this.reserveEmissionTurn("watcher");
2659
- const ids = reduceItems.map(() => this.nextDurableId(definition, "ReduceStep"));
2660
- const id = ids[ids.length - 1];
2661
- const startedAt = Date.now();
2662
- this.log("info", "durable reduce starting", {
2663
- durable_id: id,
2664
- target_function: definition.name,
2665
- collection_size: reduceItems.length,
2666
- plan_count: Math.ceil(reduceItems.length / MAX_REDUCE_CALLS_PER_PLAN)
2667
- });
2668
- const replayCallIds = ids.filter(
2669
- (_id, index) => (index + 1) % MAX_REDUCE_CALLS_PER_PLAN === 0 || index === ids.length - 1
2670
- );
2671
- for (const replayCallId of replayCallIds) this.replay?.expectCall(replayCallId);
2672
- this.replay?.expectWatcher(id);
2673
- try {
2674
- try {
2675
- await this.createReduceChain(ids, definition, reduceItems, reduceInitial, callTurn);
2676
- } catch (error) {
2677
- for (const replayCallId of replayCallIds) this.replay?.abandonCall(replayCallId);
2678
- this.replay?.abandonWatcher(id);
2679
- throw error;
2680
- }
2681
- await this.createWatcher(id, watcherTurn);
2682
- } finally {
2683
- callTurn.release();
2684
- watcherTurn.release();
2685
- }
2686
- const replayedResult = this.replay == null ? void 0 : await this.replay.takeResult(id);
2687
- const result = replayedResult ?? await this.waitForLiveEvent(
2688
- (entry) => entry.functionCallWatcherResult?.functionCallId === id
2689
- ).then((entry) => entry.functionCallWatcherResult);
2690
- let resolved;
2691
- try {
2692
- resolved = await this.resolveWatcherResult(result);
2693
- } finally {
2694
- if (replayedResult != null) this.replay?.completeResultResolution();
2695
- }
2696
- this.log("info", "durable reduce completed", {
2697
- durable_id: id,
2698
- target_function: definition.name,
2699
- collection_size: reduceItems.length,
2700
- duration_ms: Date.now() - startedAt
2701
- });
2702
- return resolved;
2703
- }
2704
- nextDurableId(definition, operation = "FunctionCall") {
2705
- const id = durableHash([
2706
- String(this.allocation.functionCallId),
2707
- this.previousDurableId,
2708
- operation,
2709
- definition.name
2710
- ]);
2711
- this.previousDurableId = id;
2712
- return id;
2713
- }
2714
- async runChild(definition, args, delaySeconds) {
2715
- const callTurn = this.reserveEmissionTurn("call");
2716
- const watcherTurn = this.reserveEmissionTurn("watcher");
2717
- const id = this.nextDurableId(definition);
2718
- const startedAt = Date.now();
2719
- const maxRetries = definition.options.retries?.maxRetries ?? this.applicationDefinition.application?.retries.maxRetries ?? 0;
2720
- this.log("info", "durable function call starting", {
2721
- durable_id: id,
2722
- target_function: definition.name,
2723
- argument_count: args.length,
2724
- delay_seconds: delaySeconds,
2725
- max_retries: maxRetries
2726
- });
2727
- this.replay?.expectCall(id);
2728
- this.replay?.expectWatcher(id);
2729
- try {
2730
- try {
2731
- await this.createChildCall(id, definition, args, delaySeconds, callTurn);
2732
- } catch (error) {
2733
- this.replay?.abandonCall(id);
2734
- this.replay?.abandonWatcher(id);
2735
- throw error;
2736
- }
2737
- await this.createWatcher(id, watcherTurn);
2738
- } finally {
2739
- callTurn.release();
2740
- watcherTurn.release();
2741
- }
2742
- this.log("debug", "waiting for terminal durable function result", {
2743
- durable_id: id,
2744
- target_function: definition.name,
2745
- max_retries: maxRetries,
2746
- replay_result_available: this.replay?.hasAvailableResult(id) ?? false
2747
- });
2748
- const replayedResult = this.replay == null ? void 0 : await this.replay.takeResult(id);
2749
- const result = replayedResult ?? await this.waitForLiveEvent(
2750
- (entry) => entry.functionCallWatcherResult?.functionCallId === id
2751
- ).then((entry) => entry.functionCallWatcherResult);
2752
- let resolved;
2753
- try {
2754
- resolved = await this.resolveWatcherResult(result);
2755
- } finally {
2756
- if (replayedResult != null) this.replay?.completeResultResolution();
2757
- }
2758
- this.log("info", "durable function call completed", {
2759
- durable_id: id,
2760
- target_function: definition.name,
2761
- max_retries: maxRetries,
2762
- duration_ms: Date.now() - startedAt
2763
- });
2764
- return resolved;
2765
- }
2766
- async createChildCall(id, definition, args, delaySeconds, turn) {
2767
- try {
2768
- let turnAcquired = false;
2769
- if (this.replay != null) {
2770
- await turn.wait;
2771
- turnAcquired = true;
2772
- }
2773
- const replayed = await this.replay?.takeOrdered("call", id);
2774
- if (replayed != null) {
2775
- this.log("debug", "replaying durable function creation", { durable_id: id });
2776
- const error2 = statusError(replayed.status, "Failed to start function call");
2777
- if (error2 != null) throw error2;
2778
- return;
2779
- }
2780
- const prepared = [];
2781
- let offset = 0;
2782
- for (const arg of args) {
2783
- const item = prepareSerializedObject(arg, offset);
2784
- prepared.push(item);
2785
- offset += item.bytes.byteLength;
2786
- }
2787
- let argsBlob;
2788
- if (prepared.length > 0) {
2789
- this.log("debug", "requesting durable function arguments blob", {
2790
- durable_id: id,
2791
- argument_count: args.length,
2792
- serialized_bytes: offset
2793
- });
2794
- try {
2795
- argsBlob = await this.requestOutputBlob(offset);
2796
- argsBlob = await this.uploadBlob(
2797
- argsBlob,
2798
- joinPrepared(prepared),
2799
- "durable function arguments",
2800
- { durable_id: id }
2801
- );
2802
- } catch (error2) {
2803
- if (this.controller.signal.aborted) throw this.controller.signal.reason;
2804
- throw this.latchProtocolError(
2805
- `Failed to materialize durable function arguments for '${id}'`,
2806
- error2
2807
- );
2808
- }
2809
- }
2810
- const metadata = Buffer.from(JSON.stringify({
2811
- format: "tensorlake.typescript.function-call.v1",
2812
- id,
2813
- functionName: definition.name,
2814
- argumentCount: args.length
2815
- }), "utf8");
2816
- const updates = {
2817
- updates: [{ functionCall: {
2818
- id,
2819
- target: {
2820
- namespace: this.functionRef.namespace,
2821
- applicationName: this.functionRef.applicationName,
2822
- applicationVersion: this.functionRef.applicationVersion,
2823
- functionName: definition.name
2824
- },
2825
- args: prepared.map((item) => ({ value: item.object })),
2826
- callMetadata: metadata
2827
- } }],
2828
- rootFunctionCallId: id
2829
- };
2830
- if (delaySeconds > 0) updates.startAt = protocolTimestamp(new Date(Date.now() + delaySeconds * 1e3));
2831
- if (!turnAcquired) await turn.wait;
2832
- this.log("debug", "queueing durable function creation event", {
2833
- durable_id: id,
2834
- target_function: definition.name,
2835
- delay_seconds: delaySeconds
2836
- });
2837
- this.addBatch([{ createFunctionCall: { updates, argsBlob } }]);
2838
- this.log("debug", "waiting for durable function creation acknowledgement", { durable_id: id });
2839
- const createdPromise = this.waitForLiveEvent((entry) => entry.functionCallCreated?.functionCallId === id);
2840
- turn.release();
2841
- const created = await createdPromise;
2842
- const error = statusError(created.functionCallCreated?.status, "Failed to start function call");
2843
- if (error != null) throw error;
2844
- } finally {
2845
- turn.release();
2846
- }
2847
- }
2848
- async createReduceChain(ids, definition, items, initial, turn) {
2849
- try {
2850
- let turnAcquired = false;
2851
- if (this.replay != null) {
2852
- await turn.wait;
2853
- turnAcquired = true;
2854
- }
2855
- const livePlans = [];
2856
- const liveRoots = [];
2857
- for (let start = 0; start < items.length; start += MAX_REDUCE_CALLS_PER_PLAN) {
2858
- const end = Math.min(items.length, start + MAX_REDUCE_CALLS_PER_PLAN);
2859
- const rootId = ids[end - 1];
2860
- const replayed = await this.replay?.takeOrdered("call", rootId);
2861
- if (replayed != null) {
2862
- this.log("debug", "replaying durable reduce plan creation", {
2863
- durable_id: rootId,
2864
- reduce_step_start: start,
2865
- reduce_step_end: end
2866
- });
2867
- const error = statusError(replayed.status, "Failed to start reduce operation");
2868
- if (error != null) throw error;
2869
- continue;
2870
- }
2871
- const prepared = [];
2872
- let offset = 0;
2873
- if (start === 0) {
2874
- const value = prepareSerializedObject(initial, offset);
2875
- prepared.push(value);
2876
- offset += value.bytes.byteLength;
2877
- }
2878
- for (let index = start; index < end; index += 1) {
2879
- const value = prepareSerializedObject(items[index], offset);
2880
- prepared.push(value);
2881
- offset += value.bytes.byteLength;
2882
- }
2883
- let argsBlob;
2884
- try {
2885
- const requested = await this.requestOutputBlob(offset);
2886
- argsBlob = await this.uploadBlob(
2887
- requested,
2888
- joinPrepared(prepared),
2889
- "durable reduce arguments",
2890
- { durable_id: rootId, reduce_step_start: start, reduce_step_end: end }
2891
- );
2892
- } catch (error) {
2893
- if (this.controller.signal.aborted) throw this.controller.signal.reason;
2894
- throw this.latchProtocolError(
2895
- `Failed to materialize durable reduce arguments for '${rootId}'`,
2896
- error
2897
- );
2898
- }
2899
- const itemOffset = start === 0 ? 1 : 0;
2900
- const updates = [];
2901
- for (let index = start; index < end; index += 1) {
2902
- const id = ids[index];
2903
- const accumulator = index === 0 ? { value: prepared[0].object } : { functionCallId: ids[index - 1] };
2904
- const item = prepared[itemOffset + index - start];
2905
- const metadata = Buffer.from(JSON.stringify({
2906
- format: "tensorlake.typescript.function-call.v1",
2907
- id,
2908
- functionName: definition.name,
2909
- argumentCount: 2,
2910
- operation: "reduce",
2911
- reduceRootId: ids[ids.length - 1],
2912
- reduceStep: index,
2913
- reduceStepCount: items.length
2914
- }), "utf8");
2915
- updates.push({ functionCall: {
2916
- id,
2917
- target: {
2918
- namespace: this.functionRef.namespace,
2919
- applicationName: this.functionRef.applicationName,
2920
- applicationVersion: this.functionRef.applicationVersion,
2921
- functionName: definition.name
2922
- },
2923
- args: [accumulator, { value: item.object }],
2924
- callMetadata: metadata
2925
- } });
2926
- }
2927
- livePlans.push({
2928
- createFunctionCall: {
2929
- updates: { updates, rootFunctionCallId: rootId },
2930
- argsBlob
2931
- }
2932
- });
2933
- liveRoots.push(rootId);
2934
- }
2935
- if (livePlans.length === 0) return;
2936
- if (!turnAcquired) await turn.wait;
2937
- this.log("debug", "queueing durable reduce function-call chains", {
2938
- durable_id: ids[ids.length - 1],
2939
- target_function: definition.name,
2940
- reduce_step_count: items.length,
2941
- plan_count: livePlans.length
2942
- });
2943
- this.addBatch(livePlans);
2944
- const creationWaiters = liveRoots.map((rootId) => {
2945
- const predicate = (entry) => entry.functionCallCreated?.functionCallId === rootId;
2946
- return {
2947
- predicate,
2948
- promise: this.waitForLiveEvent(predicate)
2949
- };
2950
- });
2951
- const createdPromises = creationWaiters.map(async ({ promise }) => {
2952
- const created = await promise;
2953
- const error = statusError(
2954
- created.functionCallCreated?.status,
2955
- "Failed to start reduce operation"
2956
- );
2957
- if (error != null) throw error;
2958
- });
2959
- turn.release();
2960
- try {
2961
- await Promise.all(createdPromises);
2962
- } catch (error) {
2963
- for (const { predicate } of creationWaiters) {
2964
- const index = this.liveWaiters.findIndex(
2965
- (waiter) => waiter.predicate === predicate
2966
- );
2967
- if (index >= 0) this.liveWaiters.splice(index, 1)[0].result.reject(error);
2968
- }
2969
- throw error;
2970
- }
2971
- } finally {
2972
- turn.release();
2973
- }
2974
- }
2975
- async createWatcher(id, turn) {
2976
- try {
2977
- await turn.wait;
2978
- const replayed = await this.replay?.takeOrdered("watcher", id);
2979
- if (replayed != null) {
2980
- this.log("debug", "replaying durable function watcher creation", { durable_id: id });
2981
- const error2 = statusError(replayed.status, "Failed to create function call watcher");
2982
- if (error2 != null) throw error2;
2983
- return;
2984
- }
2985
- this.log("debug", "queueing durable function watcher event", { durable_id: id });
2986
- this.addBatch([{ createFunctionCallWatcher: { functionCallId: id } }]);
2987
- this.log("debug", "waiting for durable function watcher acknowledgement", { durable_id: id });
2988
- const createdPromise = this.waitForLiveEvent(
2989
- (entry) => entry.functionCallWatcherCreated?.functionCallId === id
2990
- );
2991
- turn.release();
2992
- const created = await createdPromise;
2993
- const error = statusError(created.functionCallWatcherCreated?.status, "Failed to create function call watcher");
2994
- if (error != null) throw error;
2995
- } finally {
2996
- turn.release();
2997
- }
2998
- }
2999
- async resolveWatcherResult(event) {
3000
- this.log("debug", "resolving durable function watcher result", {
3001
- durable_id: event.functionCallId,
3002
- watcher_status: event.watcherStatus,
3003
- outcome_code: event.outcomeCode,
3004
- has_value_output: event.valueOutput != null,
3005
- has_request_error_output: event.requestErrorOutput != null
3006
- });
3007
- if (WATCHER_TIMED_OUT.has(event.watcherStatus)) throw new TimeoutError();
3008
- if (OUTCOME_SUCCESS.has(event.outcomeCode)) {
3009
- try {
3010
- const downloaded = await this.downloadSerializedObject(
3011
- event.valueOutput,
3012
- event.valueBlob,
3013
- "durable function result",
3014
- { durable_id: event.functionCallId }
3015
- );
3016
- return deserializeValueFromProtocol(downloaded);
3017
- } catch (error) {
3018
- if (this.controller.signal.aborted) throw this.controller.signal.reason;
3019
- throw this.latchProtocolError(
3020
- `Invalid successful function call watcher payload for '${String(event.functionCallId)}'`,
3021
- error
3022
- );
3023
- }
3024
- }
3025
- if (OUTCOME_FAILURE.has(event.outcomeCode)) {
3026
- if (event.requestErrorOutput != null) {
3027
- let message;
3028
- try {
3029
- const downloaded = await this.downloadSerializedObject(
3030
- event.requestErrorOutput,
3031
- event.requestErrorBlob,
3032
- "durable function request error",
3033
- { durable_id: event.functionCallId }
3034
- );
3035
- message = new TextDecoder("utf-8", { fatal: true }).decode(downloaded.data);
3036
- } catch (error) {
3037
- if (this.controller.signal.aborted) throw this.controller.signal.reason;
3038
- throw this.latchProtocolError(
3039
- `Invalid request-error function call watcher payload for '${String(event.functionCallId)}'`,
3040
- error
3041
- );
3042
- }
3043
- throw new RequestError(message);
3044
- }
3045
- throw new FunctionError("Function call failed");
3046
- }
3047
- throw this.latchProtocolError(
3048
- `Unexpected function call watcher outcome '${String(event.outcomeCode)}'`
3049
- );
3050
- }
3051
- latchProtocolError(message, cause) {
3052
- const protocolError = this.protocolError ?? (message instanceof AllocationProtocolError ? message : new AllocationProtocolError(
3053
- message,
3054
- cause === void 0 ? void 0 : { cause }
3055
- ));
3056
- this.protocolError = protocolError;
3057
- this.abortExecution(protocolError);
3058
- return protocolError;
3059
- }
3060
- async startTailCall(tailCall) {
3061
- const callTurn = this.reserveEmissionTurn("call");
3062
- const id = this.nextDurableId(tailCall.definition);
3063
- this.log("info", "tail call starting", {
3064
- durable_id: id,
3065
- target_function: tailCall.definition.name,
3066
- argument_count: tailCall.args.length
3067
- });
3068
- this.replay?.expectCall(id);
3069
- try {
3070
- await this.createChildCall(id, tailCall.definition, tailCall.args, 0, callTurn);
3071
- } catch (error) {
3072
- this.replay?.abandonCall(id);
3073
- throw error;
3074
- }
3075
- return id;
3076
- }
3077
- reserveEmissionTurn(kind) {
3078
- const previous = kind === "call" ? this.durableCallEmissionTail : this.durableWatcherEmissionTail;
3079
- let signalRelease;
3080
- const released = new Promise((resolve) => {
3081
- signalRelease = resolve;
3082
- });
3083
- const tail = previous.then(() => released);
3084
- if (kind === "call") this.durableCallEmissionTail = tail;
3085
- else this.durableWatcherEmissionTail = tail;
3086
- let didRelease = false;
3087
- return {
3088
- wait: previous,
3089
- release: () => {
3090
- if (didRelease) return;
3091
- didRelease = true;
3092
- signalRelease();
3093
- }
3094
- };
3095
- }
3096
- async run() {
3097
- let handlerArgs;
3098
- this.log("info", "allocation execution started", {
3099
- replay_mode: this.allocation.replayMode,
3100
- function: this.functionRef.functionName
3101
- });
3102
- this.emitLifecycleEvent("allocations_started", "Starting allocations");
3103
- try {
3104
- const inputs = this.allocation.inputs ?? {};
3105
- const objects = inputs.args ?? [];
3106
- const blobs = inputs.argBlobs ?? [];
3107
- this.log("debug", "allocation inputs validated", {
3108
- object_count: objects.length,
3109
- blob_count: blobs.length,
3110
- function_call_metadata_bytes: inputs.functionCallMetadata?.length ?? 0
3111
- });
3112
- if (objects.length !== blobs.length) throw new Error("Function argument and BLOB counts differ");
3113
- const values = await Promise.all(objects.map(
3114
- (object, index) => this.downloadSerializedObject(
3115
- object,
3116
- blobs[index],
3117
- "function input",
3118
- { input_index: index }
3119
- )
3120
- ));
3121
- this.log("info", "all function inputs downloaded", {
3122
- input_count: values.length,
3123
- total_data_bytes: values.reduce((total, value) => total + value.data.byteLength, 0)
3124
- });
3125
- const metadata = Buffer.from(inputs.functionCallMetadata ?? []);
3126
- if (metadata.byteLength === 0) {
3127
- this.log("debug", "deserializing application invocation input", {
3128
- content_type: values[0]?.contentType
3129
- });
3130
- if (values.length !== 1) throw new Error("Application calls require exactly one input payload");
3131
- try {
3132
- const parsed = await deserializeApplicationArguments(this.definition, values[0]);
3133
- handlerArgs = parsed.args;
3134
- } catch (error) {
3135
- this.log("error", "application invocation input deserialization failed", {}, error);
3136
- await this.finishUserError(error);
3137
- return;
3138
- }
3139
- this.log("debug", "application invocation input deserialized", {
3140
- handler_argument_count: handlerArgs.length
3141
- });
3142
- } else {
3143
- this.log("debug", "deserializing internal function invocation inputs", {
3144
- metadata_bytes: metadata.byteLength
3145
- });
3146
- const call = JSON.parse(metadata.toString("utf8"));
3147
- if (call.format !== "tensorlake.typescript.function-call.v1") {
3148
- throw new Error("Function call metadata was produced by a different SDK language");
3149
- }
3150
- if (call.argumentCount !== values.length) throw new Error("Function call metadata argument count differs");
3151
- try {
3152
- handlerArgs = values.map(deserializeValueFromProtocol);
3153
- } catch (error) {
3154
- this.log("error", "internal function argument deserialization failed", {}, error);
3155
- await this.finishUserError(error);
3156
- return;
3157
- }
3158
- this.log("debug", "internal function invocation inputs deserialized", {
3159
- handler_argument_count: handlerArgs.length,
3160
- metadata_function: call.functionName
3161
- });
3162
- }
3163
- if (REPLAY_STRICT.has(this.allocation.replayMode)) {
3164
- this.log("info", "strict replay history loading");
3165
- await this.loadReplayHistory();
3166
- } else {
3167
- this.log("debug", "strict replay disabled");
3168
- }
3169
- const requestContext = this.createRequestContext();
3170
- let result;
3171
- const handlerStartedAt = Date.now();
3172
- this.log("info", "user function handler starting", {
3173
- handler_argument_count: handlerArgs.length
3174
- });
3175
- try {
3176
- const execute = () => executeHandlerResult(this.definition, handlerArgs);
3177
- result = await runWithRequestContext(
3178
- requestContext,
3179
- () => runWithFunctionRuntime(this, () => waitWithAbortSignal(
3180
- this.replay?.runCausalHandler(execute) ?? execute(),
3181
- this.controller.signal
3182
- ))
3183
- );
3184
- } catch (error) {
3185
- if (error instanceof ReplayMismatchError || error instanceof AllocationProtocolError) {
3186
- throw error;
3187
- }
3188
- this.replay?.assertNoMismatch();
3189
- this.log("error", "user function handler failed", {
3190
- duration_ms: Date.now() - handlerStartedAt
3191
- }, error);
3192
- await this.finishUserError(error);
3193
- return;
3194
- }
3195
- this.log("info", "user function handler completed", {
3196
- duration_ms: Date.now() - handlerStartedAt,
3197
- result_kind: isTailCall(result) ? "tail_call" : "value",
3198
- result_type: result == null ? String(result) : typeof result
3199
- });
3200
- if (this.protocolError != null) throw this.protocolError;
3201
- this.replay?.assertNoMismatch();
3202
- if (isTailCall(result)) {
3203
- const id = await this.startTailCall(result);
3204
- this.replay?.assertConsumed();
3205
- this.controller.signal.throwIfAborted();
3206
- this.log("info", "queueing successful tail-call allocation finish", { tail_call_durable_id: id });
3207
- this.queueTerminalBatch({
3208
- outcomeCode: "ALLOCATION_OUTCOME_CODE_SUCCESS",
3209
- tailCallDurableId: id
3210
- });
3211
- } else {
3212
- this.replay?.assertConsumed();
3213
- let prepared;
3214
- try {
3215
- prepared = prepareSerializedObject(result, 0, String(this.allocation.functionCallId));
3216
- } catch (error) {
3217
- this.log("error", "function output serialization failed", {}, error);
3218
- await this.finishUserError(error);
3219
- return;
3220
- }
3221
- this.log("debug", "function output serialized", {
3222
- output_bytes: prepared.bytes.byteLength,
3223
- output_encoding: prepared.object.manifest?.encoding,
3224
- output_content_type: prepared.object.manifest?.contentType
3225
- });
3226
- let outputBlob = await this.requestOutputBlob(prepared.bytes.byteLength);
3227
- outputBlob = await this.uploadBlob(outputBlob, prepared.bytes, "function output");
3228
- this.controller.signal.throwIfAborted();
3229
- this.log("info", "queueing successful value allocation finish", {
3230
- output_blob_id: outputBlob.id,
3231
- output_bytes: prepared.bytes.byteLength
3232
- });
3233
- this.queueTerminalBatch({
3234
- outcomeCode: "ALLOCATION_OUTCOME_CODE_SUCCESS",
3235
- value: prepared.object,
3236
- uploadedFunctionOutputsBlob: outputBlob
3237
- });
3238
- }
3239
- } catch (error) {
3240
- const replayMismatch = error instanceof ReplayMismatchError;
3241
- const protocolFailure = error instanceof AllocationProtocolError || this.protocolError != null;
3242
- const cancelled = this.controller.signal.aborted && !protocolFailure;
3243
- this.log(
3244
- cancelled ? "info" : "error",
3245
- cancelled ? "allocation stopped after cancellation" : replayMismatch ? "durable replay mismatch" : "allocation internal failure",
3246
- {},
3247
- error
3248
- );
3249
- this.queueTerminalBatch({
3250
- outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
3251
- failureReason: replayMismatch ? "ALLOCATION_FAILURE_REASON_REPLAY_EVENT_HISTORY_MISMATCH" : protocolFailure ? "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR" : cancelled ? "ALLOCATION_FAILURE_REASON_FUNCTION_ERROR" : "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR"
3252
- });
3253
- } finally {
3254
- this.log("info", "allocation execution stopping", {
3255
- duration_ms: Date.now() - this.startedAt,
3256
- pending_output_blob_requests: this.outputBlobRequests.size,
3257
- pending_state_operations: this.stateOperationRequests.size,
3258
- pending_event_waiters: this.liveWaiters.length
3259
- });
3260
- this.emitLifecycleEvent("allocations_finished", "Allocations completed");
3261
- this.finish();
3262
- this.replay?.dispose();
3263
- }
3264
- }
3265
- async finishUserError(error) {
3266
- const rendered = error instanceof Error ? error.message : String(error);
3267
- this.runWithoutReplayOwnership(() => {
3268
- printCloudEvent({
3269
- level: "error",
3270
- event: "function_call_failed",
3271
- message: rendered,
3272
- namespace: this.functionRef.namespace,
3273
- application: this.functionRef.applicationName,
3274
- application_version: this.functionRef.applicationVersion,
3275
- function: this.functionRef.functionName,
3276
- request_id: this.allocation.requestId,
3277
- function_call_id: this.allocation.functionCallId,
3278
- allocation_id: this.allocation.allocationId,
3279
- error: error instanceof Error ? error.stack?.split("\n") ?? [rendered] : [rendered]
3280
- });
3281
- });
3282
- if (isRequestError(error)) {
3283
- const output = prepareTextObject(error.message);
3284
- const requestErrorBlob = this.allocation.inputs?.requestErrorBlob;
3285
- if (requestErrorBlob == null) throw new Error("Allocation has no request error BLOB");
3286
- const uploaded = await this.uploadBlob(requestErrorBlob, output.bytes, "request error output");
3287
- this.log("info", "queueing request-error allocation finish", {
3288
- request_error_bytes: output.bytes.byteLength,
3289
- request_error_blob_id: uploaded.id
3290
- });
3291
- this.queueTerminalBatch({
3292
- outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
3293
- failureReason: "ALLOCATION_FAILURE_REASON_REQUEST_ERROR",
3294
- requestErrorOutput: output.object,
3295
- uploadedRequestErrorBlob: uploaded
3296
- });
3297
- return;
3298
- }
3299
- this.log("info", "queueing function-error allocation finish", {}, error);
3300
- this.queueTerminalBatch({
3301
- outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
3302
- failureReason: "ALLOCATION_FAILURE_REASON_FUNCTION_ERROR"
3303
- });
3304
- }
3305
- createRequestContext() {
3306
- return {
3307
- requestId: String(this.allocation.requestId),
3308
- headers: new Headers2(
3309
- (this.allocation.inputs?.requestContext?.headers ?? []).map(
3310
- (header) => [
3311
- String(header.name ?? ""),
3312
- String(header.value ?? "")
3313
- ]
3314
- )
3315
- ),
3316
- signal: this.controller.signal,
3317
- state: {
3318
- get: async (key, defaultValue) => {
3319
- validateRequestStateKey(key);
3320
- const operationId = randomUUID3();
3321
- this.log("debug", "request state read starting", {
3322
- state_key: key,
3323
- operation_id: operationId,
3324
- has_default: defaultValue !== void 0
3325
- });
3326
- const result = await this.requestStateOperation({ operationId, stateKey: key, prepareRead: {} });
3327
- if (result.status?.code === NOT_FOUND) {
3328
- this.log("debug", "request state key not found", { state_key: key, operation_id: operationId });
3329
- return defaultValue;
3330
- }
3331
- const error = statusError(result.status, `Failed to read request state '${key}'`);
3332
- if (error != null) throw error;
3333
- const data = await this.downloadBlob(result.prepareRead?.blob, "request state value", {
3334
- state_key: key,
3335
- operation_id: operationId
3336
- });
3337
- const value = deserializeJSON(data);
3338
- this.log("debug", "request state read completed", {
3339
- state_key: key,
3340
- operation_id: operationId,
3341
- value_bytes: data.byteLength
3342
- });
3343
- return value;
3344
- },
3345
- set: async (key, value) => {
3346
- validateRequestStateKey(key);
3347
- const serialized = serializeRequestStateValue(value);
3348
- const prepareOperationId = randomUUID3();
3349
- this.log("debug", "request state write preparation starting", {
3350
- state_key: key,
3351
- operation_id: prepareOperationId,
3352
- value_bytes: serialized.byteLength
3353
- });
3354
- const prepared = await this.requestStateOperation({
3355
- operationId: prepareOperationId,
3356
- stateKey: key,
3357
- prepareWrite: { size: serialized.byteLength }
3358
- });
3359
- const error = statusError(prepared.status, `Failed to prepare request state '${key}'`);
3360
- if (error != null) throw error;
3361
- const blob = await this.uploadBlob(prepared.prepareWrite?.blob, serialized, "request state value", {
3362
- state_key: key,
3363
- operation_id: prepareOperationId
3364
- });
3365
- const commitOperationId = randomUUID3();
3366
- this.log("debug", "request state write commit starting", {
3367
- state_key: key,
3368
- operation_id: commitOperationId,
3369
- blob_id: blob.id
3370
- });
3371
- const committed = await this.requestStateOperation({
3372
- operationId: commitOperationId,
3373
- stateKey: key,
3374
- commitWrite: { blob }
3375
- });
3376
- const commitError = statusError(committed.status, `Failed to commit request state '${key}'`);
3377
- if (commitError != null) throw commitError;
3378
- this.log("debug", "request state write completed", {
3379
- state_key: key,
3380
- operation_id: commitOperationId,
3381
- value_bytes: serialized.byteLength
3382
- });
3383
- }
3384
- },
3385
- metrics: {
3386
- counter: async (name, value = 1) => {
3387
- validateCounterMetric(name, value);
3388
- this.emitUserEvent("ai.tensorlake.metric.counter.inc", {
3389
- counter_name: name,
3390
- counter_inc: value
3391
- });
3392
- },
3393
- timer: async (name, value) => {
3394
- validateTimerMetric(name, value);
3395
- this.emitUserEvent("ai.tensorlake.metric.timer", {
3396
- timer_name: name,
3397
- timer_value: value
3398
- });
3399
- }
3400
- },
3401
- progress: {
3402
- update: async (current, total, options) => {
3403
- validateProgressUpdate(current, total, options);
3404
- this.log("debug", "request progress update", {
3405
- current,
3406
- total,
3407
- has_custom_message: options?.message != null,
3408
- attribute_count: Object.keys(options?.attributes ?? {}).length
3409
- });
3410
- this.state.progress = { current, total };
3411
- this.publishState();
3412
- const message = options?.message ?? `${this.functionRef.functionName}: executing step ${current} of ${total}`;
3413
- this.runWithoutReplayOwnership(() => {
3414
- printCloudEvent({ RequestProgressUpdated: {
3415
- request_id: this.allocation.requestId,
3416
- function_name: this.functionRef.functionName,
3417
- function_run_id: this.allocation.functionCallId,
3418
- allocation_id: this.allocation.allocationId,
3419
- message,
3420
- step: current,
3421
- total,
3422
- attributes: options?.attributes,
3423
- created_at: (/* @__PURE__ */ new Date()).toISOString()
3424
- } }, {
3425
- type: "ai.tensorlake.progress_update",
3426
- source: "/tensorlake/applications/progress",
3427
- message
3428
- });
3429
- });
3430
- }
3431
- }
3432
- };
3433
- }
3434
- emitUserEvent(type, data) {
3435
- this.log("debug", "user metric event emitted", {
3436
- event_type: type,
3437
- metric_name: data.counter_name ?? data.timer_name
3438
- });
3439
- this.runWithoutReplayOwnership(() => {
3440
- printCloudEvent({
3441
- request_id: this.allocation.requestId,
3442
- function_name: this.functionRef.functionName,
3443
- function_run_id: this.allocation.functionCallId,
3444
- allocation_id: this.allocation.allocationId,
3445
- ...data
3446
- }, { type, source: "/tensorlake/applications/metrics" });
3447
- });
3448
- }
3449
- emitLifecycleEvent(event, message) {
3450
- this.runWithoutReplayOwnership(() => {
3451
- printCloudEvent({
3452
- event,
3453
- message,
3454
- allocations: [{
3455
- namespace: this.functionRef.namespace,
3456
- application: this.functionRef.applicationName,
3457
- application_version: this.functionRef.applicationVersion,
3458
- function: this.functionRef.functionName,
3459
- request_id: this.allocation.requestId,
3460
- function_call_id: this.allocation.functionCallId,
3461
- allocation_id: this.allocation.allocationId
3462
- }]
3463
- });
3464
- });
3465
- }
3466
- async loadReplayHistory() {
3467
- const entries = [];
3468
- let afterClock = 0;
3469
- let page = 0;
3470
- while (true) {
3471
- page += 1;
3472
- this.log("debug", "requesting replay history page", { page, after_clock: afterClock });
3473
- const requestedAfterClock = afterClock;
3474
- const response = await this.readEvents(afterClock);
3475
- entries.push(...response.entries ?? []);
3476
- this.log("debug", "replay history page received", {
3477
- page,
3478
- entry_count: response.entries?.length ?? 0,
3479
- last_clock: response.lastClock,
3480
- has_more: response.hasMore
3481
- });
3482
- const responseClock = validatedEventLogClock(
3483
- response,
3484
- requestedAfterClock,
3485
- (message) => new ReplayMismatchError(message)
3486
- );
3487
- afterClock = responseClock;
3488
- if (!response.hasMore) break;
3489
- }
3490
- this.lastEventClock = afterClock;
3491
- this.replay = new ReplayHistory(entries);
3492
- this.log("info", "strict replay history loaded", {
3493
- page_count: page,
3494
- entry_count: entries.length,
3495
- last_clock: afterClock
3496
- });
3497
- }
3498
- waitForLiveEvent(predicate) {
3499
- if (this.controller.signal.aborted) {
3500
- return Promise.reject(this.controller.signal.reason);
3501
- }
3502
- const index = this.liveBacklog.findIndex(predicate);
3503
- if (index >= 0) {
3504
- this.log("debug", "durable event satisfied from backlog", {
3505
- backlog_size: this.liveBacklog.length,
3506
- pending_waiters: this.liveWaiters.length
3507
- });
3508
- return Promise.resolve(this.liveBacklog.splice(index, 1)[0]);
3509
- }
3510
- const result = deferred();
3511
- this.liveWaiters.push({ predicate, result });
3512
- this.log("debug", "waiting for durable event", {
3513
- backlog_size: this.liveBacklog.length,
3514
- pending_waiters: this.liveWaiters.length
3515
- });
3516
- this.ensureLivePump();
3517
- return result.promise;
3518
- }
3519
- ensureLivePump() {
3520
- if (this.livePump != null) {
3521
- this.log("debug", "durable event live pump already running");
3522
- return;
3523
- }
3524
- this.log("info", "durable event live pump starting", { after_clock: this.lastEventClock });
3525
- const pump = (async () => {
3526
- while (!this.finished && this.liveWaiters.length > 0) {
3527
- const requestedAfterClock = this.lastEventClock;
3528
- const response = await this.readEvents(this.lastEventClock);
3529
- this.log("debug", "durable event log page received", {
3530
- requested_after_clock: requestedAfterClock,
3531
- entry_count: response.entries?.length ?? 0,
3532
- last_clock: response.lastClock,
3533
- has_more: response.hasMore
3534
- });
3535
- const responseClock = validatedEventLogClock(
3536
- response,
3537
- requestedAfterClock,
3538
- (message) => new AllocationProtocolError(message)
3539
- );
3540
- this.lastEventClock = responseClock;
3541
- for (const entry of response.entries ?? []) this.dispatchLiveEvent(entry);
3542
- }
3543
- })();
3544
- this.livePump = pump;
3545
- void pump.then(() => {
3546
- if (this.livePump !== pump) return;
3547
- this.livePump = void 0;
3548
- this.log("debug", "durable event live pump stopped", {
3549
- finished: this.finished,
3550
- pending_waiters: this.liveWaiters.length,
3551
- after_clock: this.lastEventClock
3552
- });
3553
- if (!this.finished && this.liveWaiters.length > 0) this.ensureLivePump();
3554
- }, (error) => {
3555
- if (this.livePump === pump) this.livePump = void 0;
3556
- if (!this.finished && !this.controller.signal.aborted) {
3557
- const protocolError = error instanceof AllocationProtocolError ? error : new AllocationProtocolError(
3558
- error instanceof Error ? error.message : String(error),
3559
- { cause: error }
3560
- );
3561
- this.latchProtocolError(protocolError);
3562
- this.log("error", "durable event live pump failed", {
3563
- pending_waiters: this.liveWaiters.length
3564
- }, protocolError);
3565
- }
3566
- });
3567
- }
3568
- dispatchLiveEvent(entry) {
3569
- const index = this.liveWaiters.findIndex((waiter) => waiter.predicate(entry));
3570
- const eventKind = Object.keys(entry).find((key) => key !== "clock") ?? "unknown";
3571
- if (index < 0) {
3572
- this.liveBacklog.push(entry);
3573
- this.log("debug", "durable event added to backlog", {
3574
- event_kind: eventKind,
3575
- event_clock: entry.clock,
3576
- backlog_size: this.liveBacklog.length
3577
- });
3578
- } else {
3579
- this.liveWaiters.splice(index, 1)[0].result.resolve(entry);
3580
- this.log("debug", "durable event delivered to waiter", {
3581
- event_kind: eventKind,
3582
- event_clock: entry.clock,
3583
- pending_waiters: this.liveWaiters.length
3584
- });
3585
- }
3586
- }
3587
- readEvents(afterClock) {
3588
- if (this.currentRead != null) {
3589
- throw new AllocationProtocolError("Allocation event log already has a pending read");
3590
- }
3591
- const response = deferred();
3592
- const request = {
3593
- allocationId: this.allocation.allocationId,
3594
- afterClock,
3595
- maxEntries: 100
3596
- };
3597
- this.currentRead = { request, response };
3598
- this.log("debug", "event log read requested", {
3599
- after_clock: afterClock,
3600
- max_entries: request.maxEntries,
3601
- connected_streams: this.eventReadStreams.size
3602
- });
3603
- for (const stream of [...this.eventReadStreams]) {
3604
- try {
3605
- stream.write(request);
3606
- } catch (error) {
3607
- this.dropEventReadStream(stream, "write_failed", error);
3608
- }
3609
- }
3610
- return response.promise;
3611
- }
3612
- watchEventLogReads(stream) {
3613
- this.eventReadStreams.add(stream);
3614
- this.log("info", "event log read stream connected", {
3615
- connected_streams: this.eventReadStreams.size,
3616
- has_pending_read: this.currentRead != null
3617
- });
3618
- try {
3619
- stream.on("cancelled", () => this.dropEventReadStream(stream, "cancelled"));
3620
- stream.on("close", () => this.dropEventReadStream(stream, "closed"));
3621
- if (this.currentRead != null) {
3622
- this.log("debug", "sending pending event log read to new stream", {
3623
- after_clock: this.currentRead.request.afterClock,
3624
- max_entries: this.currentRead.request.maxEntries
3625
- });
3626
- stream.write(this.currentRead.request);
3627
- }
3628
- if (this.finished && this.currentRead == null) stream.end();
3629
- } catch (error) {
3630
- this.dropEventReadStream(stream, "setup_failed", error);
3631
- throw error;
3632
- }
3633
- }
3634
- deliverEventLogResponse(response) {
3635
- if (this.currentRead == null) {
3636
- this.log("warn", "unexpected event log response ignored", {
3637
- entry_count: response.entries?.length ?? 0,
3638
- last_clock: response.lastClock
3639
- });
3640
- return;
3641
- }
3642
- const pending = this.currentRead;
3643
- this.currentRead = void 0;
3644
- this.log("debug", "event log response delivered", {
3645
- requested_after_clock: pending.request.afterClock,
3646
- entry_count: response.entries?.length ?? 0,
3647
- last_clock: response.lastClock,
3648
- has_more: response.hasMore
3649
- });
3650
- pending.response.resolve(response);
3651
- }
3652
- requestOutputBlob(size) {
3653
- if (this.controller.signal.aborted) {
3654
- return Promise.reject(this.controller.signal.reason);
3655
- }
3656
- const id = randomUUID3();
3657
- const request = deferred();
3658
- this.outputBlobRequests.set(id, request);
3659
- this.state.outputBlobRequests.push({ id, size });
3660
- this.log("debug", "output blob requested", {
3661
- blob_id: id,
3662
- requested_bytes: size,
3663
- pending_output_blob_requests: this.outputBlobRequests.size
3664
- });
3665
- this.publishState();
3666
- return request.promise.then((output) => {
3667
- this.log("debug", "output blob response received", {
3668
- blob_id: id,
3669
- status_code: output.status?.code,
3670
- chunk_count: output.blob?.chunks?.length ?? 0
3671
- });
3672
- const error = statusError(output.status, "Failed to create output BLOB");
3673
- if (error != null) throw error;
3674
- return output.blob;
3675
- });
3676
- }
3677
- requestStateOperation(operation) {
3678
- if (this.controller.signal.aborted) {
3679
- return Promise.reject(this.controller.signal.reason);
3680
- }
3681
- const request = deferred();
3682
- this.stateOperationRequests.set(operation.operationId, request);
3683
- this.state.requestStateOperations.push(operation);
3684
- this.log("debug", "request state operation queued", {
3685
- operation_id: operation.operationId,
3686
- state_key: operation.stateKey,
3687
- operation_kind: this.stateOperationKind(operation),
3688
- requested_bytes: operation.prepareWrite?.size,
3689
- pending_state_operations: this.stateOperationRequests.size
3690
- });
3691
- this.publishState();
3692
- return request.promise;
3693
- }
3694
- deliverUpdate(update) {
3695
- if (update.outputBlob != null) {
3696
- let id = String(update.outputBlob.blob?.id ?? "");
3697
- let matchedByOrder = false;
3698
- if (!id) {
3699
- id = String(this.state.outputBlobRequests[0]?.id ?? "");
3700
- matchedByOrder = id !== "";
3701
- }
3702
- const request = this.outputBlobRequests.get(id);
3703
- if (request == null) {
3704
- this.log("warn", "unmatched output blob update ignored", {
3705
- blob_id: id || void 0,
3706
- status_code: update.outputBlob.status?.code,
3707
- pending_output_blob_requests: this.outputBlobRequests.size
3708
- });
3709
- return;
3710
- }
3711
- this.outputBlobRequests.delete(id);
3712
- this.state.outputBlobRequests = this.state.outputBlobRequests.filter((item) => item.id !== id);
3713
- this.log("debug", "output blob update matched", {
3714
- blob_id: id,
3715
- status_code: update.outputBlob.status?.code,
3716
- chunk_count: update.outputBlob.blob?.chunks?.length ?? 0,
3717
- matched_by_order: matchedByOrder,
3718
- pending_output_blob_requests: this.outputBlobRequests.size
3719
- });
3720
- request.resolve(update.outputBlob);
3721
- this.publishState();
3722
- return;
3723
- }
3724
- const result = update.requestStateOperationResult;
3725
- if (result != null) {
3726
- if (typeof result.operationId !== "string" || result.operationId.length === 0) {
3727
- throw this.latchProtocolError(
3728
- "Allocation received a request state operation result without an operation ID"
3729
- );
3730
- }
3731
- const id = result.operationId;
3732
- const request = this.stateOperationRequests.get(id);
3733
- if (request == null) {
3734
- this.log("warn", "unmatched request state operation result ignored", {
3735
- operation_id: id,
3736
- status_code: result.status?.code,
3737
- pending_state_operations: this.stateOperationRequests.size
3738
- });
3739
- return;
3740
- }
3741
- const operation = this.state.requestStateOperations.find((item) => item.operationId === id);
3742
- this.stateOperationRequests.delete(id);
3743
- this.state.requestStateOperations = this.state.requestStateOperations.filter(
3744
- (item) => item.operationId !== id
3745
- );
3746
- this.log("debug", "request state operation result matched", {
3747
- operation_id: id,
3748
- state_key: operation?.stateKey,
3749
- operation_kind: this.stateOperationKind(operation),
3750
- status_code: result.status?.code,
3751
- pending_state_operations: this.stateOperationRequests.size
3752
- });
3753
- request.resolve(result);
3754
- this.publishState();
3755
- return;
3756
- }
3757
- this.log("warn", "allocation update contained no recognized payload");
3758
- }
3759
- watchState(stream) {
3760
- this.stateStreams.add(stream);
3761
- this.stateStreamFlows.set(stream, { backpressured: false });
3762
- this.log("info", "allocation state stream connected", {
3763
- connected_streams: this.stateStreams.size,
3764
- state_hash: this.state.sha256Hash,
3765
- pending_output_blob_requests: this.outputBlobRequests.size,
3766
- pending_state_operations: this.stateOperationRequests.size
3767
- });
3768
- try {
3769
- stream.on("cancelled", () => this.dropStateStream(stream, "cancelled"));
3770
- stream.on("close", () => this.dropStateStream(stream, "closed"));
3771
- stream.on("drain", () => this.drainStateStream(stream));
3772
- this.writeStateSnapshot(stream, this.snapshotState());
3773
- if (this.finished) stream.end();
3774
- } catch (error) {
3775
- this.dropStateStream(stream, "setup_failed", error);
3776
- throw error;
3777
- }
3778
- }
3779
- updateStateHash() {
3780
- const content = JSON.stringify({
3781
- progress: this.state.progress,
3782
- outputBlobRequests: this.state.outputBlobRequests,
3783
- requestStateOperations: this.state.requestStateOperations
3784
- });
3785
- this.state.sha256Hash = createHash3("sha256").update(content).digest("hex");
3786
- }
3787
- snapshotState() {
3788
- return nativeStructuredClone(this.state);
3789
- }
3790
- publishState() {
3791
- this.updateStateHash();
3792
- this.log("debug", "allocation state published", {
3793
- state_hash: this.state.sha256Hash,
3794
- connected_streams: this.stateStreams.size,
3795
- pending_output_blob_requests: this.outputBlobRequests.size,
3796
- pending_state_operations: this.stateOperationRequests.size,
3797
- progress_current: this.state.progress?.current,
3798
- progress_total: this.state.progress?.total
3799
- });
3800
- for (const stream of [...this.stateStreams]) {
3801
- this.writeStateSnapshot(stream, this.snapshotState());
3802
- }
3803
- }
3804
- writeStateSnapshot(stream, snapshot) {
3805
- const flow = this.stateStreamFlows.get(stream);
3806
- if (flow == null) return;
3807
- if (flow.backpressured) {
3808
- flow.pending = snapshot;
3809
- return;
3810
- }
3811
- try {
3812
- flow.backpressured = !stream.write(snapshot);
3813
- } catch (error) {
3814
- this.dropStateStream(stream, "write_failed", error);
3815
- }
3816
- }
3817
- drainStateStream(stream) {
3818
- const flow = this.stateStreamFlows.get(stream);
3819
- if (flow == null) return;
3820
- flow.backpressured = false;
3821
- const pending = flow.pending;
3822
- flow.pending = void 0;
3823
- if (pending != null) this.writeStateSnapshot(stream, pending);
3824
- }
3825
- addBatch(events) {
3826
- if (this.terminalBatchQueued && events.every((event) => event.finishAllocation == null)) {
3827
- this.log("error", "execution event rejected after terminal batch", {
3828
- event_count: events.length,
3829
- event_kinds: events.map((event) => Object.keys(event)[0] ?? "unknown")
3830
- });
3831
- return;
3832
- }
3833
- this.executionBatches.push(events);
3834
- this.log("debug", "execution log batch queued", {
3835
- event_count: events.length,
3836
- event_kinds: events.map((event) => Object.keys(event)[0] ?? "unknown"),
3837
- queued_batches: this.executionBatches.length,
3838
- waiting_consumers: this.executionBatchWaiters.length
3839
- });
3840
- for (const waiter of this.executionBatchWaiters.splice(0)) {
3841
- this.markTerminalBatchObserved(events);
3842
- waiter(events);
3843
- }
3844
- }
3845
- queueTerminalBatch(finishAllocation) {
3846
- if (this.terminalBatchQueued) {
3847
- this.log("error", "duplicate terminal allocation batch rejected", {
3848
- outcome_code: finishAllocation.outcomeCode,
3849
- failure_reason: finishAllocation.failureReason
3850
- });
3851
- return;
3852
- }
3853
- this.terminalBatchQueued = true;
3854
- this.addBatch([{ finishAllocation }]);
3855
- }
3856
- async getExecutionBatch() {
3857
- if (this.executionBatches.length > 0) {
3858
- this.log("debug", "returning queued execution log batch", {
3859
- event_count: this.executionBatches[0].length,
3860
- queued_batches: this.executionBatches.length
3861
- });
3862
- const events = this.executionBatches[0];
3863
- this.markTerminalBatchObserved(events);
3864
- return events;
3865
- }
3866
- if (this.finished) {
3867
- this.log("debug", "returning empty execution log batch for finished allocation");
3868
- return [];
3869
- }
3870
- this.log("debug", "execution log batch consumer waiting", {
3871
- waiting_consumers: this.executionBatchWaiters.length + 1
3872
- });
3873
- return new Promise((resolve) => this.executionBatchWaiters.push(resolve));
3874
- }
3875
- advanceExecutionBatch() {
3876
- const advanced = this.executionBatches.shift();
3877
- this.log("debug", "execution log batch advanced", {
3878
- advanced_event_count: advanced?.length ?? 0,
3879
- queued_batches: this.executionBatches.length
3880
- });
3881
- }
3882
- markTerminalBatchObserved(events) {
3883
- if (this.terminalBatchObserved || !events.some((event) => event.finishAllocation != null)) {
3884
- return;
3885
- }
3886
- this.terminalBatchObserved = true;
3887
- this.log("debug", "terminal execution log batch observed by consumer");
3888
- for (const waiter of this.terminalBatchObservedWaiters.splice(0)) waiter();
3889
- }
3890
- get isFinished() {
3891
- return this.finished;
3892
- }
3893
- finish() {
3894
- if (this.finished) {
3895
- this.log("debug", "allocation finish ignored because runner is already finished");
3896
- return;
3897
- }
3898
- this.finished = true;
3899
- this.abortExecution(new FunctionError("Allocation finished"));
3900
- this.log("info", "allocation runner finished", {
3901
- duration_ms: Date.now() - this.startedAt,
3902
- queued_execution_batches: this.executionBatches.length,
3903
- state_streams: this.stateStreams.size,
3904
- event_log_streams: this.eventReadStreams.size
3905
- });
3906
- for (const stream of [...this.stateStreams]) {
3907
- try {
3908
- stream.end();
3909
- } catch (error) {
3910
- this.dropStateStream(stream, "end_failed", error);
3911
- }
3912
- }
3913
- for (const stream of [...this.eventReadStreams]) {
3914
- try {
3915
- stream.end();
3916
- } catch (error) {
3917
- this.dropEventReadStream(stream, "end_failed", error);
3918
- }
3919
- }
3920
- for (const waiter of this.executionBatchWaiters.splice(0)) waiter([]);
3921
- }
3922
- dropStateStream(stream, reason, error) {
3923
- if (!this.stateStreams.delete(stream)) return;
3924
- this.stateStreamFlows.delete(stream);
3925
- this.log("info", "allocation state stream disconnected", {
3926
- reason,
3927
- connected_streams: this.stateStreams.size
3928
- }, error);
3929
- }
3930
- dropEventReadStream(stream, reason, error) {
3931
- if (!this.eventReadStreams.delete(stream)) return;
3932
- this.log("info", "event log read stream disconnected", {
3933
- reason,
3934
- connected_streams: this.eventReadStreams.size,
3935
- has_pending_read: this.currentRead != null
3936
- }, error);
3937
- }
3938
- stateOperationKind(operation) {
3939
- if (operation?.prepareRead != null) return "prepare_read";
3940
- if (operation?.prepareWrite != null) return "prepare_write";
3941
- if (operation?.commitWrite != null) return "commit_write";
3942
- return "unknown";
3943
- }
3944
- blobLogFields(blob) {
3945
- const chunks = blob?.chunks ?? [];
3946
- return {
3947
- blob_id: blob?.id,
3948
- chunk_count: chunks.length,
3949
- declared_bytes: chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0)
3950
- };
3951
- }
3952
- async downloadBlob(blob, purpose, fields = {}) {
3953
- if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
3954
- const startedAt = Date.now();
3955
- this.log("debug", "blob download starting", {
3956
- purpose,
3957
- ...fields,
3958
- ...this.blobLogFields(blob)
3959
- });
3960
- try {
3961
- const data = await downloadBlob(
3962
- blob,
3963
- (message, details) => this.log("debug", `blob ${message}`, {
3964
- purpose,
3965
- ...fields,
3966
- ...details
3967
- }),
3968
- this.controller.signal
3969
- );
3970
- this.log("debug", "blob download completed", {
3971
- purpose,
3972
- ...fields,
3973
- ...this.blobLogFields(blob),
3974
- downloaded_bytes: data.byteLength,
3975
- duration_ms: Date.now() - startedAt
3976
- });
3977
- return data;
3978
- } catch (error) {
3979
- this.log("error", "blob download failed", {
3980
- purpose,
3981
- ...fields,
3982
- ...this.blobLogFields(blob),
3983
- duration_ms: Date.now() - startedAt
3984
- }, error);
3985
- throw error;
3986
- }
3987
- }
3988
- async downloadSerializedObject(object, blob, purpose, fields = {}) {
3989
- if (object == null) throw new Error(`Missing serialized object for ${purpose}`);
3990
- if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
3991
- const startedAt = Date.now();
3992
- this.log("debug", "serialized object download starting", {
3993
- purpose,
3994
- ...fields,
3995
- ...this.blobLogFields(blob),
3996
- object_offset: object.offset ?? 0,
3997
- object_bytes: object.manifest?.size,
3998
- metadata_bytes: object.manifest?.metadataSize,
3999
- content_type: object.manifest?.contentType,
4000
- encoding: object.manifest?.encoding
4001
- });
4002
- try {
4003
- const value = await downloadSerializedObject(
4004
- object,
4005
- blob,
4006
- (message, details) => this.log(
4007
- "debug",
4008
- `blob ${message}`,
4009
- { purpose, ...fields, ...details }
4010
- ),
4011
- this.controller.signal
4012
- );
4013
- this.log("debug", "serialized object download completed", {
4014
- purpose,
4015
- ...fields,
4016
- ...this.blobLogFields(blob),
4017
- object_bytes: value.data.byteLength + value.metadata.byteLength,
4018
- data_bytes: value.data.byteLength,
4019
- metadata_bytes: value.metadata.byteLength,
4020
- duration_ms: Date.now() - startedAt
4021
- });
4022
- return value;
4023
- } catch (error) {
4024
- this.log("error", "serialized object download failed", {
4025
- purpose,
4026
- ...fields,
4027
- ...this.blobLogFields(blob),
4028
- duration_ms: Date.now() - startedAt
4029
- }, error);
4030
- throw error;
4031
- }
4032
- }
4033
- async uploadBlob(blob, data, purpose, fields = {}) {
4034
- if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
4035
- const startedAt = Date.now();
4036
- this.log("debug", "blob upload starting", {
4037
- purpose,
4038
- ...fields,
4039
- ...this.blobLogFields(blob),
4040
- upload_bytes: data.byteLength
4041
- });
4042
- try {
4043
- const uploaded = await uploadBlob(
4044
- blob,
4045
- data,
4046
- (message, details) => this.log("debug", `blob ${message}`, {
4047
- purpose,
4048
- ...fields,
4049
- ...details
4050
- }),
4051
- this.controller.signal
4052
- );
4053
- this.log("debug", "blob upload completed", {
4054
- purpose,
4055
- ...fields,
4056
- ...this.blobLogFields(uploaded),
4057
- upload_bytes: data.byteLength,
4058
- duration_ms: Date.now() - startedAt
4059
- });
4060
- return uploaded;
4061
- } catch (error) {
4062
- this.log("error", "blob upload failed", {
4063
- purpose,
4064
- ...fields,
4065
- ...this.blobLogFields(blob),
4066
- upload_bytes: data.byteLength,
4067
- duration_ms: Date.now() - startedAt
4068
- }, error);
4069
- throw error;
4070
- }
4071
- }
4072
- log(level, message, fields = {}, error) {
4073
- this.runWithoutReplayOwnership(() => {
4074
- writeStructuredOutput("stderr", () => {
4075
- const rendered = error instanceof Error ? error.stack?.split("\n") ?? [`${error.name}: ${error.message}`] : error == null ? void 0 : [String(error)];
4076
- return {
4077
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4078
- level,
4079
- component: "typescript_function_executor_allocation",
4080
- message,
4081
- allocation_id: this.allocation.allocationId,
4082
- request_id: this.allocation.requestId,
4083
- function_call_id: this.allocation.functionCallId,
4084
- namespace: this.functionRef.namespace,
4085
- application: this.functionRef.applicationName,
4086
- application_version: this.functionRef.applicationVersion,
4087
- function: this.functionRef.functionName,
4088
- elapsed_ms: Date.now() - this.startedAt,
4089
- ...fields,
4090
- ...rendered == null ? {} : { error: rendered }
4091
- };
4092
- });
4093
- });
4094
- }
4095
- runWithoutReplayOwnership(callback) {
4096
- return this.replay?.runWithoutCausalOwnership(callback) ?? callback();
4097
- }
4098
- };
4099
-
4100
- // src/function-executor/service.ts
4101
- var SHUTDOWN_TERMINAL_DRAIN_TIMEOUT_MS = 1e3;
4102
- var nativeImport = compileFunction(
4103
- "return import(specifier)",
4104
- ["specifier"],
4105
- { importModuleDynamically: vmConstants.USE_MAIN_CONTEXT_DEFAULT_LOADER }
4106
- );
4107
- function rpcError(code, details) {
4108
- return Object.assign(new Error(details), { code, details, metadata: new grpc.Metadata() });
4109
- }
4110
- function requireSafeSize(value, field) {
4111
- if (!Number.isSafeInteger(value) || Number(value) < 0) {
4112
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field} must be a non-negative safe integer`);
4113
- }
4114
- return Number(value);
4115
- }
4116
- function validateSerializedObjectManifest(manifest, field) {
4117
- if (manifest == null) {
4118
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest is required`);
4119
- }
4120
- if (manifest.encoding == null) {
4121
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.encoding is required`);
4122
- }
4123
- if (manifest.encodingVersion == null) {
4124
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.encoding_version is required`);
4125
- }
4126
- requireSafeSize(manifest.encodingVersion, `${field}.manifest.encoding_version`);
4127
- requireSafeSize(manifest.size, `${field}.manifest.size`);
4128
- if (manifest.metadataSize == null) {
4129
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.metadata_size is required`);
4130
- }
4131
- const metadataSize = requireSafeSize(manifest.metadataSize, `${field}.manifest.metadata_size`);
4132
- if (metadataSize > Number(manifest.size)) {
4133
- throw rpcError(
4134
- grpc.status.INVALID_ARGUMENT,
4135
- `${field}.manifest.metadata_size cannot exceed size`
4136
- );
4137
- }
4138
- if (typeof manifest.sha256Hash !== "string" || manifest.sha256Hash.length === 0) {
4139
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.sha256_hash is required`);
4140
- }
4141
- }
4142
- function validateBlob(blob, field) {
4143
- if (blob == null) throw rpcError(grpc.status.INVALID_ARGUMENT, `${field} is required`);
4144
- if (!Array.isArray(blob.chunks)) {
4145
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.chunks must be an array`);
4146
- }
4147
- blob.chunks.forEach((chunk, index) => {
4148
- if (chunk == null || typeof chunk.uri !== "string" || chunk.uri.length === 0) {
4149
- throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.chunks[${index}].uri is required`);
4150
- }
4151
- requireSafeSize(chunk.size, `${field}.chunks[${index}].size`);
4152
- });
4153
- }
4154
- function validateNewAllocation(allocation) {
4155
- if (allocation == null) {
4156
- throw rpcError(grpc.status.INVALID_ARGUMENT, "CreateAllocationRequest.allocation is required");
4157
- }
4158
- for (const [field, value] of [
4159
- ["request_id", allocation.requestId],
4160
- ["function_call_id", allocation.functionCallId],
4161
- ["allocation_id", allocation.allocationId]
4162
- ]) {
4163
- if (typeof value !== "string" || value.length === 0) {
4164
- throw rpcError(grpc.status.INVALID_ARGUMENT, `Allocation.${field} is required`);
4165
- }
4166
- }
4167
- const inputs = allocation.inputs;
4168
- if (inputs == null) {
4169
- throw rpcError(grpc.status.INVALID_ARGUMENT, "Allocation.inputs is required");
4170
- }
4171
- if (!Array.isArray(inputs.args) || !Array.isArray(inputs.argBlobs)) {
4172
- throw rpcError(grpc.status.INVALID_ARGUMENT, "Allocation input args and arg_blobs must be arrays");
4173
- }
4174
- if (inputs.args.length !== inputs.argBlobs.length) {
4175
- throw rpcError(
4176
- grpc.status.INVALID_ARGUMENT,
4177
- `Mismatched function arguments and argument blobs lengths, ${inputs.args.length} != ${inputs.argBlobs.length}`
4178
- );
4179
- }
4180
- inputs.args.forEach((argument, index) => {
4181
- if (argument == null) {
4182
- throw rpcError(grpc.status.INVALID_ARGUMENT, `Allocation.inputs.args[${index}] is required`);
4183
- }
4184
- requireSafeSize(argument.offset, `Allocation.inputs.args[${index}].offset`);
4185
- validateSerializedObjectManifest(argument.manifest, `Allocation.inputs.args[${index}]`);
4186
- validateBlob(inputs.argBlobs[index], `Allocation.inputs.arg_blobs[${index}]`);
4187
- });
4188
- validateBlob(inputs.requestErrorBlob, "Allocation.inputs.request_error_blob");
4189
- }
4190
- function safeArchivePath(root, archivePath) {
4191
- const output = path.resolve(root, archivePath);
4192
- if (output !== root && !output.startsWith(`${root}${path.sep}`)) {
4193
- throw new Error(`Application archive contains unsafe path '${archivePath}'`);
4194
- }
4195
- return output;
4196
- }
4197
- async function unpackApplicationCode(data) {
4198
- const root = await mkdtemp(path.join(os.tmpdir(), "tensorlake-typescript-app-"));
4199
- try {
4200
- const files = unzipSync(data);
4201
- for (const [name, content] of Object.entries(files)) {
4202
- const destination = safeArchivePath(root, name);
4203
- await mkdir(path.dirname(destination), { recursive: true });
4204
- await writeFile(destination, content, { mode: 384 });
4205
- }
4206
- const manifest = JSON.parse(
4207
- await readFile(path.join(root, ".tensorlake_code_manifest.json"), "utf8")
4208
- );
4209
- return { root, manifest };
4210
- } catch (error) {
4211
- await rm(root, { recursive: true, force: true });
4212
- throw error;
4213
- }
4214
- }
4215
- var FunctionExecutorService = class {
4216
- allocations = /* @__PURE__ */ new Map();
4217
- functionRef;
4218
- definition;
4219
- applicationDefinition;
4220
- initialization;
4221
- stopping = false;
4222
- shutdownPromise;
4223
- implementation = {
4224
- initialize: this.initialize.bind(this),
4225
- listAllocations: this.listAllocations.bind(this),
4226
- createAllocation: this.createAllocation.bind(this),
4227
- deleteAllocation: this.deleteAllocation.bind(this),
4228
- watchAllocationState: this.watchAllocationState.bind(this),
4229
- sendAllocationUpdate: this.sendAllocationUpdate.bind(this),
4230
- getAllocationExecutionLogBatch: this.getAllocationExecutionLogBatch.bind(this),
4231
- advanceAllocationExecutionLogBatch: this.advanceAllocationExecutionLogBatch.bind(this),
4232
- watchAllocationEventLogReads: this.watchAllocationEventLogReads.bind(this),
4233
- sendAllocationEventLogReadResponse: this.sendAllocationEventLogReadResponse.bind(this),
4234
- checkHealth: this.checkHealth.bind(this),
4235
- getInfo: this.getInfo.bind(this)
4236
- };
4237
- shutdown(reason = new Error("Function executor is shutting down")) {
4238
- if (this.shutdownPromise != null) return this.shutdownPromise;
4239
- this.stopping = true;
4240
- const shutdown = (async () => {
4241
- this.log("info", "function executor service shutdown requested", {
4242
- allocation_count: this.allocations.size
4243
- });
4244
- const allocations = [...this.allocations.values()];
4245
- for (const allocation of allocations) allocation.cancel(reason);
4246
- await Promise.all(allocations.map((allocation) => allocation.waitForCompletion()));
4247
- const terminalObservations = await Promise.all(
4248
- allocations.map(
4249
- (allocation) => allocation.waitForTerminalBatchObserved(SHUTDOWN_TERMINAL_DRAIN_TIMEOUT_MS)
4250
- )
4251
- );
4252
- const unobservedCount = terminalObservations.filter((observed) => !observed).length;
4253
- if (unobservedCount > 0) {
4254
- this.log(
4255
- "warn",
4256
- "shutdown terminal execution batches were not observed before drain deadline",
4257
- { unobserved_allocation_count: unobservedCount }
4258
- );
4259
- }
4260
- this.log("info", "function executor service shutdown completed", {
4261
- allocation_count: allocations.length
4262
- });
4263
- })();
4264
- this.shutdownPromise = shutdown;
4265
- return shutdown;
4266
- }
4267
- initialize(call, callback) {
4268
- if (this.stopping) {
4269
- callback(rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down"));
4270
- return;
4271
- }
4272
- const startedAt = Date.now();
4273
- this.log("info", "initialize RPC received", {
4274
- ...this.functionLogFields(call.request.function),
4275
- application_code_bytes: call.request.applicationCode?.data?.length
4276
- });
4277
- this.emitInitializationEvent(call.request.function, "function_executor_initialization_started", "Initializing function executor");
4278
- void this.initializeAsync(call.request).then(
4279
- () => {
4280
- this.log("info", "initialize RPC completed", {
4281
- ...this.functionLogFields(call.request.function),
4282
- duration_ms: Date.now() - startedAt,
4283
- outcome: "success"
4284
- });
4285
- this.emitInitializationEvent(call.request.function, "function_executor_initialization_finished", "Function executor initialization completed");
4286
- callback(null, { outcomeCode: "INITIALIZATION_OUTCOME_CODE_SUCCESS" });
4287
- },
4288
- (error) => {
4289
- const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
4290
- this.log("error", "initialize RPC failed", {
4291
- ...this.functionLogFields(call.request.function),
4292
- duration_ms: Date.now() - startedAt,
4293
- outcome: "failure"
4294
- }, error);
4295
- this.emitInitializationEvent(
4296
- call.request.function,
4297
- "function_executor_initialization_failed",
4298
- "Function executor initialization failed",
4299
- error
4300
- );
4301
- callback(null, {
4302
- outcomeCode: "INITIALIZATION_OUTCOME_CODE_FAILURE",
4303
- failureReason: "INITIALIZATION_FAILURE_REASON_FUNCTION_ERROR",
4304
- errorMessage: message
4305
- });
4306
- }
4307
- );
4308
- }
4309
- emitInitializationEvent(functionRef, event, message, error) {
4310
- const rendered = error instanceof Error ? error.stack?.split("\n") : error == null ? void 0 : [String(error)];
4311
- printCloudEvent({
4312
- ...error == null ? {} : { level: "error" },
4313
- event,
4314
- message,
4315
- namespace: functionRef?.namespace,
4316
- application: functionRef?.applicationName,
4317
- application_version: functionRef?.applicationVersion,
4318
- function: functionRef?.functionName,
4319
- ...rendered == null ? {} : { error: rendered }
4320
- });
4321
- }
4322
- async initializeAsync(request) {
4323
- if (this.definition != null) throw new Error("Function Executor is already initialized");
4324
- if (this.initialization != null) throw new Error("Function Executor initialization is already in progress");
4325
- const initialization = this.initializeAttempt(request);
4326
- this.initialization = initialization;
4327
- try {
4328
- await initialization;
4329
- } finally {
4330
- if (this.initialization === initialization) this.initialization = void 0;
4331
- }
4332
- }
4333
- async initializeAttempt(request) {
4334
- if (request.function == null) throw new Error("InitializeRequest.function is required");
4335
- for (const [field, value] of [
4336
- ["namespace", request.function.namespace],
4337
- ["application_name", request.function.applicationName],
4338
- ["function_name", request.function.functionName],
4339
- ["application_version", request.function.applicationVersion]
4340
- ]) {
4341
- if (typeof value !== "string" || value.length === 0) {
4342
- throw new Error(`InitializeRequest.function.${field} is required`);
4343
- }
4344
- }
4345
- if (request.applicationCode == null) throw new Error("InitializeRequest.application_code is required");
4346
- if (request.applicationCode.manifest == null) throw new Error("InitializeRequest.application_code.manifest is required");
4347
- if (request.applicationCode.data == null) throw new Error("InitializeRequest.application_code.data is required");
4348
- const codeManifest = request.applicationCode.manifest;
4349
- if (codeManifest.encoding == null) throw new Error("InitializeRequest.application_code.manifest.encoding is required");
4350
- if (codeManifest.encodingVersion == null) throw new Error("InitializeRequest.application_code.manifest.encoding_version is required");
4351
- if (codeManifest.size == null) throw new Error("InitializeRequest.application_code.manifest.size is required");
4352
- if (codeManifest.sha256Hash == null) throw new Error("InitializeRequest.application_code.manifest.sha256_hash is required");
4353
- if (!(/* @__PURE__ */ new Set(["SERIALIZED_OBJECT_ENCODING_BINARY_ZIP", 4])).has(codeManifest.encoding)) {
4354
- throw new Error(
4355
- `Invalid application code encoding: ${String(codeManifest.encoding)}. Expected: BINARY_ZIP`
4356
- );
4357
- }
4358
- const declaredSize = requireSafeSize(
4359
- codeManifest.size,
4360
- "InitializeRequest.application_code.manifest.size"
4361
- );
4362
- const metadataSize = codeManifest.metadataSize == null ? 0 : requireSafeSize(
4363
- codeManifest.metadataSize,
4364
- "InitializeRequest.application_code.manifest.metadata_size"
4365
- );
4366
- if (metadataSize > declaredSize) {
4367
- throw new Error(
4368
- "InitializeRequest.application_code.manifest.metadata_size cannot exceed size"
4369
- );
4370
- }
4371
- const archive = new Uint8Array(request.applicationCode.data);
4372
- this.log("debug", "unpacking application code", {
4373
- ...this.functionLogFields(request.function),
4374
- application_code_bytes: archive.byteLength
4375
- });
4376
- const unpackStartedAt = Date.now();
4377
- const { root, manifest } = await unpackApplicationCode(archive);
4378
- let registrySnapshot;
4379
- try {
4380
- this.log("debug", "application code unpacked", {
4381
- ...this.functionLogFields(request.function),
4382
- duration_ms: Date.now() - unpackStartedAt,
4383
- manifest_format_version: manifest.format_version,
4384
- manifest_runtime: manifest.runtime,
4385
- manifest_function_count: Object.keys(manifest.functions ?? {}).length
4386
- });
4387
- if (manifest.format_version !== 2 || manifest.runtime !== "typescript") {
4388
- throw new Error("Application code is not a Tensorlake TypeScript bundle");
4389
- }
4390
- const nodeMajor = Number(process.versions.node.split(".")[0]);
4391
- if ((manifest.minimum_node_major ?? 24) > nodeMajor) {
4392
- throw new Error(`Application requires Node ${manifest.minimum_node_major} or newer; executor has ${nodeMajor}`);
4393
- }
4394
- const moduleName = manifest.module;
4395
- if (!moduleName) throw new Error("Application code manifest does not name a runtime module");
4396
- const functionName = String(request.function.functionName);
4397
- if (manifest.functions?.[functionName] == null) {
4398
- throw new Error(`Function '${functionName}' is not present in the application bundle`);
4399
- }
4400
- const modulePath = safeArchivePath(root, moduleName);
4401
- this.log("debug", "importing application runtime module", {
4402
- ...this.functionLogFields(request.function),
4403
- module: moduleName
4404
- });
4405
- const importStartedAt = Date.now();
4406
- registrySnapshot = snapshotRegistry();
4407
- const runtime = await nativeImport(`${pathToFileURL(modulePath).href}?executor=${Date.now()}`);
4408
- if (typeof runtime.__tensorlakeGetFunction !== "function") {
4409
- throw new Error("Application runtime does not export __tensorlakeGetFunction");
4410
- }
4411
- const definition = runtime.__tensorlakeGetFunction(functionName);
4412
- if (definition?.name !== functionName) {
4413
- throw new Error(
4414
- `Application runtime resolved function '${functionName}' as '${String(definition?.name)}'`
4415
- );
4416
- }
4417
- const applicationName = String(request.function.applicationName);
4418
- const applicationDefinition = runtime.__tensorlakeGetFunction(applicationName);
4419
- if (applicationDefinition?.name !== applicationName) {
4420
- throw new Error(
4421
- `Application runtime resolved application '${applicationName}' as '${String(applicationDefinition?.name)}'`
4422
- );
4423
- }
4424
- if (applicationDefinition.application == null) {
4425
- throw new Error(`'${applicationName}' is not a Tensorlake application`);
4426
- }
4427
- this.functionRef = request.function;
4428
- this.definition = definition;
4429
- this.applicationDefinition = applicationDefinition;
4430
- this.log("debug", "application function resolved", {
4431
- ...this.functionLogFields(request.function),
4432
- duration_ms: Date.now() - importStartedAt,
4433
- parameter_count: definition.parameters.length,
4434
- application_max_retries: applicationDefinition.application.retries.maxRetries
4435
- });
4436
- registrySnapshot = void 0;
4437
- } catch (error) {
4438
- if (registrySnapshot != null) restoreRegistry(registrySnapshot);
4439
- this.functionRef = void 0;
4440
- this.definition = void 0;
4441
- this.applicationDefinition = void 0;
4442
- await rm(root, { recursive: true, force: true });
4443
- throw error;
4444
- }
4445
- }
4446
- getAllocation(id) {
4447
- const allocation = this.allocations.get(id);
4448
- if (allocation == null) throw rpcError(grpc.status.NOT_FOUND, `Allocation ${id} not found`);
4449
- return allocation;
4450
- }
4451
- listAllocations(_call, callback) {
4452
- this.log("debug", "list allocations RPC received", { allocation_count: this.allocations.size });
4453
- callback(null, { allocations: [...this.allocations.values()].map((runner) => runner.allocation) });
4454
- }
4455
- createAllocation(call, callback) {
4456
- void this.createAllocationAsync(call).then(
4457
- () => callback(null, {}),
4458
- (error) => {
4459
- this.log(
4460
- "error",
4461
- "create allocation RPC failed",
4462
- this.allocationLogFields(call.request.allocation),
4463
- error
4464
- );
4465
- callback(error);
4466
- }
4467
- );
4468
- }
4469
- async createAllocationAsync(call) {
4470
- if (this.stopping) {
4471
- throw rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down");
4472
- }
4473
- const initialization = this.initialization;
4474
- if (initialization != null) {
4475
- this.log("info", "create allocation RPC waiting for initialization", {
4476
- ...this.allocationLogFields(call.request.allocation)
4477
- });
4478
- await this.waitForInitialization(call, initialization);
4479
- this.log("info", "create allocation RPC resumed after initialization", {
4480
- ...this.allocationLogFields(call.request.allocation)
4481
- });
4482
- }
4483
- this.assertAllocationAdmissionActive(call);
4484
- if (this.stopping) {
4485
- throw rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down");
4486
- }
4487
- if (this.definition == null || this.applicationDefinition == null || this.functionRef == null) {
4488
- throw rpcError(grpc.status.FAILED_PRECONDITION, "Function Executor is not initialized");
4489
- }
4490
- const allocation = call.request.allocation;
4491
- validateNewAllocation(allocation);
4492
- const id = String(allocation.allocationId);
4493
- this.log("info", "create allocation RPC received", {
4494
- ...this.allocationLogFields(allocation),
4495
- replay_mode: allocation?.replayMode,
4496
- input_count: allocation?.inputs?.args?.length ?? 0
4497
- });
4498
- if (this.allocations.has(id)) throw rpcError(grpc.status.ALREADY_EXISTS, `Allocation ${id} already exists`);
4499
- const runner = new AllocationRunner(
4500
- allocation,
4501
- this.functionRef,
4502
- this.definition,
4503
- this.applicationDefinition
4504
- );
4505
- this.allocations.set(id, runner);
4506
- runner.start();
4507
- this.log("info", "allocation runner started", this.allocationLogFields(allocation));
4508
- }
4509
- waitForInitialization(call, initialization) {
4510
- return new Promise((resolve, reject) => {
4511
- let timer;
4512
- let settled = false;
4513
- const deadline = typeof call.getDeadline === "function" ? call.getDeadline() : Infinity;
4514
- const deadlineMs = deadline instanceof Date ? deadline.getTime() : Number(deadline);
4515
- const cancellationError = () => Number.isFinite(deadlineMs) && deadlineMs <= Date.now() ? rpcError(
4516
- grpc.status.DEADLINE_EXCEEDED,
4517
- "Initialization did not finish before the allocation deadline"
4518
- ) : rpcError(
4519
- grpc.status.CANCELLED,
4520
- "Create allocation RPC was cancelled during initialization"
4521
- );
4522
- const onCancelled = () => finish(cancellationError());
4523
- const finish = (error) => {
4524
- if (settled) return;
4525
- settled = true;
4526
- if (timer != null) clearTimeout(timer);
4527
- if (typeof call.off === "function") call.off("cancelled", onCancelled);
4528
- if (error == null) resolve();
4529
- else reject(error);
4530
- };
4531
- if (call.cancelled || Number.isFinite(deadlineMs) && deadlineMs <= Date.now()) {
4532
- finish(cancellationError());
4533
- return;
4534
- }
4535
- if (typeof call.once === "function") call.once("cancelled", onCancelled);
4536
- if (Number.isFinite(deadlineMs)) {
4537
- const remainingMs = deadlineMs - Date.now();
4538
- if (remainingMs <= 0) {
4539
- finish(rpcError(grpc.status.DEADLINE_EXCEEDED, "Initialization did not finish before the allocation deadline"));
4540
- return;
4541
- }
4542
- timer = setTimeout(() => {
4543
- finish(rpcError(
4544
- grpc.status.DEADLINE_EXCEEDED,
4545
- "Initialization did not finish before the allocation deadline"
4546
- ));
4547
- }, remainingMs);
4548
- }
4549
- initialization.then(
4550
- () => finish(
4551
- call.cancelled || Number.isFinite(deadlineMs) && deadlineMs <= Date.now() ? cancellationError() : void 0
4552
- ),
4553
- () => finish(rpcError(
4554
- grpc.status.FAILED_PRECONDITION,
4555
- "Function Executor initialization failed"
4556
- ))
4557
- );
4558
- });
4559
- }
4560
- assertAllocationAdmissionActive(call) {
4561
- const deadline = typeof call.getDeadline === "function" ? call.getDeadline() : Infinity;
4562
- const deadlineMs = deadline instanceof Date ? deadline.getTime() : Number(deadline);
4563
- if (Number.isFinite(deadlineMs) && deadlineMs <= Date.now()) {
4564
- throw rpcError(grpc.status.DEADLINE_EXCEEDED, "Create allocation RPC deadline exceeded");
4565
- }
4566
- if (call.cancelled) {
4567
- throw rpcError(grpc.status.CANCELLED, "Create allocation RPC was cancelled before allocation start");
4568
- }
4569
- }
4570
- deleteAllocation(call, callback) {
4571
- try {
4572
- const id = String(call.request.allocationId ?? "");
4573
- const allocation = this.getAllocation(id);
4574
- this.log("debug", "delete allocation RPC received", {
4575
- allocation_id: id,
4576
- finished: allocation.isFinished
4577
- });
4578
- if (!allocation.isFinished) {
4579
- throw rpcError(grpc.status.FAILED_PRECONDITION, `Allocation ${id} is still running`);
4580
- }
4581
- this.allocations.delete(id);
4582
- this.log("info", "allocation deleted", { allocation_id: id });
4583
- callback(null, {});
4584
- } catch (error) {
4585
- this.log("error", "delete allocation RPC failed", { allocation_id: call.request.allocationId }, error);
4586
- callback(error);
4587
- }
4588
- }
4589
- watchAllocationState(call) {
4590
- try {
4591
- const allocationId = String(call.request.allocationId ?? "");
4592
- this.log("debug", "allocation state stream opening", { allocation_id: allocationId });
4593
- this.getAllocation(allocationId).watchState(call);
4594
- } catch (error) {
4595
- this.log("error", "allocation state stream failed to open", { allocation_id: call.request.allocationId }, error);
4596
- call.destroy(error);
4597
- }
4598
- }
4599
- sendAllocationUpdate(call, callback) {
4600
- try {
4601
- const allocationId = String(call.request.allocationId ?? "");
4602
- this.log("debug", "allocation update RPC received", {
4603
- allocation_id: allocationId,
4604
- update_kind: call.request.outputBlob != null ? "output_blob" : call.request.requestStateOperationResult != null ? "request_state_operation_result" : "unknown"
4605
- });
4606
- if (call.request.outputBlob == null && call.request.requestStateOperationResult == null) {
4607
- throw rpcError(
4608
- grpc.status.INVALID_ARGUMENT,
4609
- `Unknown update type in AllocationUpdate for allocation ${allocationId}`
4610
- );
4611
- }
4612
- const allocation = this.getAllocation(allocationId);
4613
- if (allocation.isFinished) throw rpcError(grpc.status.FAILED_PRECONDITION, "Allocation is already finished");
4614
- allocation.deliverUpdate(call.request);
4615
- callback(null, {});
4616
- } catch (error) {
4617
- this.log("error", "allocation update RPC failed", { allocation_id: call.request.allocationId }, error);
4618
- callback(error);
4619
- }
4620
- }
4621
- getAllocationExecutionLogBatch(call, callback) {
4622
- try {
4623
- const allocationId = String(call.request.allocationId ?? "");
4624
- this.log("debug", "execution log batch RPC waiting", { allocation_id: allocationId });
4625
- const allocation = this.getAllocation(allocationId);
4626
- void allocation.getExecutionBatch().then(
4627
- (events) => {
4628
- this.log("debug", "execution log batch RPC completed", {
4629
- allocation_id: allocationId,
4630
- event_count: events.length
4631
- });
4632
- callback(null, { events });
4633
- },
4634
- (error) => {
4635
- this.log("error", "execution log batch RPC failed", { allocation_id: allocationId }, error);
4636
- callback(error);
4637
- }
4638
- );
4639
- } catch (error) {
4640
- this.log("error", "execution log batch RPC failed", { allocation_id: call.request.allocationId }, error);
4641
- callback(error);
4642
- }
4643
- }
4644
- advanceAllocationExecutionLogBatch(call, callback) {
4645
- try {
4646
- const allocationId = String(call.request.allocationId ?? "");
4647
- this.log("debug", "advance execution log batch RPC received", { allocation_id: allocationId });
4648
- this.getAllocation(allocationId).advanceExecutionBatch();
4649
- callback(null, {});
4650
- } catch (error) {
4651
- this.log("error", "advance execution log batch RPC failed", { allocation_id: call.request.allocationId }, error);
4652
- callback(error);
4653
- }
4654
- }
4655
- watchAllocationEventLogReads(call) {
4656
- try {
4657
- const allocationId = String(call.request.allocationId ?? "");
4658
- const allocation = this.getAllocation(allocationId);
4659
- this.log("debug", "event log read stream opening", { allocation_id: allocationId });
4660
- call.sendMetadata(new grpc.Metadata());
4661
- this.log("debug", "event log read stream established", { allocation_id: allocationId });
4662
- allocation.watchEventLogReads(call);
4663
- } catch (error) {
4664
- this.log("error", "event log read stream failed to open", { allocation_id: call.request.allocationId }, error);
4665
- call.destroy(error);
4666
- }
4667
- }
4668
- sendAllocationEventLogReadResponse(call, callback) {
4669
- try {
4670
- const allocationId = String(call.request.allocationId ?? "");
4671
- this.log("debug", "event log read response RPC received", {
4672
- allocation_id: allocationId,
4673
- entry_count: call.request.entries?.length ?? 0,
4674
- last_clock: call.request.lastClock,
4675
- has_more: call.request.hasMore
4676
- });
4677
- this.getAllocation(allocationId).deliverEventLogResponse(call.request);
4678
- callback(null, {});
4679
- } catch (error) {
4680
- this.log("error", "event log read response RPC failed", { allocation_id: call.request.allocationId }, error);
4681
- callback(error);
4682
- }
4683
- }
4684
- checkHealth(_call, callback) {
4685
- this.log("debug", "health check RPC received", { initialized: this.definition != null });
4686
- if (this.definition == null) {
4687
- callback(rpcError(grpc.status.UNAVAILABLE, "Function Executor is not initialized"));
4688
- } else {
4689
- callback(null, { healthy: true, statusMessage: "ok" });
4690
- }
4691
- }
4692
- getInfo(_call, callback) {
4693
- this.log("debug", "executor info RPC received", { protocol_version: "0.1.3" });
4694
- callback(null, {
4695
- version: "0.1.3",
4696
- sdkVersion: SDK_VERSION,
4697
- sdkLanguage: "typescript",
4698
- sdkLanguageVersion: process.versions.node
4699
- });
4700
- }
4701
- functionLogFields(functionRef) {
4702
- return {
4703
- namespace: functionRef?.namespace,
4704
- application: functionRef?.applicationName,
4705
- application_version: functionRef?.applicationVersion,
4706
- function: functionRef?.functionName
4707
- };
4708
- }
4709
- allocationLogFields(allocation) {
4710
- return {
4711
- allocation_id: allocation?.allocationId,
4712
- request_id: allocation?.requestId,
4713
- function_call_id: allocation?.functionCallId
4714
- };
4715
- }
4716
- log(level, message, fields = {}, error) {
4717
- writeStructuredOutput("stderr", () => {
4718
- const rendered = error instanceof Error ? error.stack?.split("\n") ?? [`${error.name}: ${error.message}`] : error == null ? void 0 : [String(error)];
4719
- return {
4720
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4721
- level,
4722
- component: "typescript_function_executor_service",
4723
- message,
4724
- ...fields,
4725
- ...rendered == null ? {} : { error: rendered }
4726
- };
4727
- });
4728
- }
4729
- };
4730
-
4731
- // src/function-executor/main.ts
4732
- var SHUTDOWN_GRACE_PERIOD_MS = 1e4;
4733
- var IPV4_LOOPBACK_ADDRESS = "127.0.0.1";
4734
- function functionExecutorBindAddress(address) {
4735
- if (address === "localhost") return IPV4_LOOPBACK_ADDRESS;
4736
- if (address.startsWith("localhost:")) {
4737
- return `${IPV4_LOOPBACK_ADDRESS}${address.slice("localhost".length)}`;
4738
- }
4739
- return address;
4740
- }
4741
- function parseArguments(argv) {
4742
- const result = { functionExecutorId: "" };
4743
- for (let index = 0; index < argv.length; index += 1) {
4744
- const argument = argv[index];
4745
- const [key, inline] = argument.split("=", 2);
4746
- const value = inline ?? argv[index + 1];
4747
- if (key === "--address") result.address = value;
4748
- else if (key === "--executor-id") result.executorId = value;
4749
- else if (key === "--function-executor-id") result.functionExecutorId = value ?? "";
4750
- else continue;
4751
- if (inline == null) index += 1;
4752
- }
4753
- return result;
4754
- }
4755
- function serviceDefinition() {
4756
- const protoRoot = functionExecutorProtoRoot();
4757
- const protoPath = path2.join(protoRoot, "tensorlake/function_executor/proto/function_executor.proto");
4758
- const packageDefinition = loadSync(protoPath, {
4759
- includeDirs: [path2.dirname(protoPath), protoRoot, getProtoPath()],
4760
- keepCase: false,
4761
- longs: Number,
4762
- enums: String,
4763
- defaults: false,
4764
- oneofs: true
4765
- });
4766
- const loaded = grpc2.loadPackageDefinition(packageDefinition);
4767
- return loaded.function_executor_service.FunctionExecutor.service;
4768
- }
4769
- function functionExecutorProtoRoot() {
4770
- const relativeRoots = [
4771
- // Standalone function-executor capsule.
4772
- "../../proto/",
4773
- // Function executor launched directly from the published SDK package.
4774
- "../../runtime/function-executor/package/proto/",
4775
- // Development checkout: both language runtimes use the repository source.
4776
- "../../../proto/"
4777
- ];
4778
- for (const relativeRoot of relativeRoots) {
4779
- const protoRoot = fileURLToPath2(new URL(relativeRoot, import.meta.url));
4780
- const protoPath = path2.join(protoRoot, "tensorlake/function_executor/proto/function_executor.proto");
4781
- if (existsSync(protoPath)) return protoRoot;
4782
- }
4783
- throw new Error("Cannot find the shared Tensorlake function-executor protobuf definition");
4784
- }
4785
- async function main(argv = process.argv.slice(2)) {
4786
- configureNativeWorker(() => new URL("../native-worker.cjs", import.meta.url));
4787
- const nodeMajor = Number(process.versions.node.split(".")[0]);
4788
- if (nodeMajor < 24) throw new Error(`Tensorlake TypeScript functions require Node 24 or newer; got ${process.versions.node}`);
4789
- const args = parseArguments(argv);
4790
- if (!args.address) throw new Error("--address argument is required");
4791
- if (!args.executorId) throw new Error("--executor-id argument is required");
4792
- const address = functionExecutorBindAddress(args.address);
4793
- writeStructuredOutput("stderr", () => ({
4794
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4795
- level: "info",
4796
- component: "typescript_function_executor_main",
4797
- message: "starting TypeScript function executor",
4798
- address,
4799
- executor_id: args.executorId,
4800
- fn_executor_id: args.functionExecutorId,
4801
- node_version: process.versions.node,
4802
- pid: process.pid
4803
- }));
4804
- const server = new grpc2.Server({
4805
- "grpc.max_receive_message_length": -1,
4806
- "grpc.max_send_message_length": -1,
4807
- "grpc.so_reuseport": 0
4808
- });
4809
- const service = new FunctionExecutorService();
4810
- server.addService(serviceDefinition(), service.implementation);
4811
- await new Promise((resolve, reject) => {
4812
- server.bindAsync(address, grpc2.ServerCredentials.createInsecure(), (error) => {
4813
- if (error != null) reject(error);
4814
- else resolve();
4815
- });
4816
- });
4817
- writeStructuredOutput("stderr", () => ({
4818
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4819
- level: "info",
4820
- component: "typescript_function_executor_main",
4821
- message: "started TypeScript function executor",
4822
- address,
4823
- executor_id: args.executorId,
4824
- fn_executor_id: args.functionExecutorId,
4825
- node_version: process.versions.node
4826
- }));
4827
- let stopping = false;
4828
- const shutdown = (signal) => {
4829
- if (stopping) return;
4830
- stopping = true;
4831
- writeStructuredOutput("stderr", () => ({
4832
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4833
- level: "info",
4834
- component: "typescript_function_executor_main",
4835
- message: "stopping TypeScript function executor",
4836
- signal,
4837
- executor_id: args.executorId,
4838
- fn_executor_id: args.functionExecutorId,
4839
- pid: process.pid
4840
- }));
4841
- const forceShutdownTimer = setTimeout(() => {
4842
- writeStructuredOutput("stderr", () => ({
4843
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4844
- level: "error",
4845
- component: "typescript_function_executor_main",
4846
- message: "forcing TypeScript function executor shutdown after grace period",
4847
- signal,
4848
- executor_id: args.executorId,
4849
- fn_executor_id: args.functionExecutorId,
4850
- pid: process.pid,
4851
- grace_period_ms: SHUTDOWN_GRACE_PERIOD_MS
4852
- }));
4853
- server.forceShutdown();
4854
- process.exit(1);
4855
- }, SHUTDOWN_GRACE_PERIOD_MS);
4856
- const gracefulShutdown = service.shutdown(new Error(`Function executor received ${signal}`)).then(() => new Promise((resolve, reject) => {
4857
- server.tryShutdown((error) => {
4858
- if (error != null) reject(error);
4859
- else resolve();
4860
- });
4861
- }));
4862
- void gracefulShutdown.then(
4863
- () => {
4864
- clearTimeout(forceShutdownTimer);
4865
- writeStructuredOutput("stderr", () => ({
4866
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4867
- level: "info",
4868
- component: "typescript_function_executor_main",
4869
- message: "stopped TypeScript function executor",
4870
- signal,
4871
- executor_id: args.executorId,
4872
- fn_executor_id: args.functionExecutorId,
4873
- pid: process.pid
4874
- }));
4875
- process.exit(0);
4876
- },
4877
- (error) => {
4878
- clearTimeout(forceShutdownTimer);
4879
- writeStructuredOutput("stderr", () => ({
4880
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
4881
- level: "error",
4882
- component: "typescript_function_executor_main",
4883
- message: "TypeScript function executor shutdown failed",
4884
- signal,
4885
- executor_id: args.executorId,
4886
- fn_executor_id: args.functionExecutorId,
4887
- pid: process.pid,
4888
- error: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
4889
- }));
4890
- server.forceShutdown();
4891
- process.exit(1);
4892
- }
4893
- );
4894
- };
4895
- process.once("SIGINT", () => shutdown("SIGINT"));
4896
- process.once("SIGTERM", () => shutdown("SIGTERM"));
4897
- }
4898
- export {
4899
- functionExecutorBindAddress,
4900
- main
4901
- };
4902
- //# sourceMappingURL=main.js.map