tensorlake 0.5.142 → 0.5.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -159
- package/applications.cjs +3 -0
- package/applications.d.cts +1 -0
- package/applications.d.ts +1 -0
- package/applications.js +1 -0
- package/index.cjs +3 -0
- package/index.d.cts +1 -0
- package/index.d.ts +1 -0
- package/index.js +1 -0
- package/package.json +25 -68
- package/bin/function-executor.js +0 -8
- package/bin/tensorlake-create-sandbox-image.js +0 -9
- package/bin/tensorlake-deploy.cjs +0 -8
- package/bin/tensorlake-import-sandbox-image.js +0 -9
- package/bin/tensorlake-typescript-function-runner.js +0 -8
- package/dist/applications/index.cjs +0 -4812
- package/dist/applications/index.cjs.map +0 -1
- package/dist/applications/index.d.cts +0 -130
- package/dist/applications/index.d.ts +0 -130
- package/dist/applications/index.js +0 -4773
- package/dist/applications/index.js.map +0 -1
- package/dist/function-agent/main.cjs +0 -1838
- package/dist/function-agent/main.cjs.map +0 -1
- package/dist/function-agent/main.d.cts +0 -3
- package/dist/function-agent/main.d.ts +0 -3
- package/dist/function-agent/main.js +0 -1825
- package/dist/function-agent/main.js.map +0 -1
- package/dist/function-executor/main.cjs +0 -4944
- package/dist/function-executor/main.cjs.map +0 -1
- package/dist/function-executor/main.d.cts +0 -4
- package/dist/function-executor/main.d.ts +0 -4
- package/dist/function-executor/main.js +0 -4902
- package/dist/function-executor/main.js.map +0 -1
- package/dist/index.cjs +0 -10175
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.cts +0 -2427
- package/dist/index.d.ts +0 -2427
- package/dist/index.js +0 -10099
- package/dist/index.js.map +0 -1
- package/dist/native-worker.cjs +0 -278
- package/dist/native-worker.cjs.map +0 -1
- package/dist/native-worker.d.cts +0 -2
- package/dist/native-worker.d.ts +0 -2
- package/dist/native-worker.js +0 -276
- package/dist/native-worker.js.map +0 -1
- package/dist/remote-D95Y6Czr.d.cts +0 -282
- package/dist/remote-omwTzbfj.d.ts +0 -282
- package/dist/sandbox-image-ChzWDeZy.d.cts +0 -357
- package/dist/sandbox-image-ChzWDeZy.d.ts +0 -357
- package/dist/sandbox-image.cjs +0 -1992
- package/dist/sandbox-image.cjs.map +0 -1
- package/dist/sandbox-image.d.cts +0 -1
- package/dist/sandbox-image.d.ts +0 -1
- package/dist/sandbox-image.js +0 -1987
- package/dist/sandbox-image.js.map +0 -1
- package/lib/libc.cjs +0 -61
- package/lib/libc.d.cts +0 -1
- package/lib/runtime.cjs +0 -155
- package/runtime/function-executor/manifest.json +0 -59
- package/runtime/function-executor/package/bin/function-executor.js +0 -8
- package/runtime/function-executor/package/dist/function-executor/main.js +0 -4902
- package/runtime/function-executor/package/dist/function-executor/main.js.map +0 -1
- package/runtime/function-executor/package/dist/native-worker.cjs +0 -278
- package/runtime/function-executor/package/dist/native-worker.cjs.map +0 -1
- package/runtime/function-executor/package/npm-shrinkwrap.json +0 -475
- package/runtime/function-executor/package/package.json +0 -22
- package/runtime/function-executor/package/proto/README.md +0 -18
- package/runtime/function-executor/package/proto/tensorlake/function_executor/proto/function_executor.proto +0 -613
- package/runtime/function-executor/package/proto/tensorlake/function_executor/proto/status.proto +0 -42
- package/runtime/typescript-function-runner/manifest.json +0 -44
- package/runtime/typescript-function-runner/package/bin/tensorlake-typescript-function-runner.js +0 -8
- package/runtime/typescript-function-runner/package/dist/function-agent/main.js +0 -1825
- package/runtime/typescript-function-runner/package/dist/function-agent/main.js.map +0 -1
- package/runtime/typescript-function-runner/package/dist/native-worker.cjs +0 -278
- package/runtime/typescript-function-runner/package/dist/native-worker.cjs.map +0 -1
- package/runtime/typescript-function-runner/package/npm-shrinkwrap.json +0 -580
- package/runtime/typescript-function-runner/package/package.json +0 -25
|
@@ -1,4944 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
|
|
30
|
-
// src/function-executor/main.ts
|
|
31
|
-
var main_exports = {};
|
|
32
|
-
__export(main_exports, {
|
|
33
|
-
functionExecutorBindAddress: () => functionExecutorBindAddress,
|
|
34
|
-
main: () => main
|
|
35
|
-
});
|
|
36
|
-
module.exports = __toCommonJS(main_exports);
|
|
37
|
-
|
|
38
|
-
// node_modules/tsup/assets/cjs_shims.js
|
|
39
|
-
var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.tagName.toUpperCase() === "SCRIPT" ? document.currentScript.src : new URL("main.js", document.baseURI).href;
|
|
40
|
-
var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
41
|
-
|
|
42
|
-
// src/function-executor/main.ts
|
|
43
|
-
var import_node_fs = require("fs");
|
|
44
|
-
|
|
45
|
-
// src/native-worker-client.ts
|
|
46
|
-
var import_node_module = require("module");
|
|
47
|
-
var import_node_worker_threads = require("worker_threads");
|
|
48
|
-
|
|
49
|
-
// src/native-worker-protocol.ts
|
|
50
|
-
var cancelNativeCall = /* @__PURE__ */ Symbol.for("tensorlake.native.cancel.v1");
|
|
51
|
-
var closeNativeHandle = /* @__PURE__ */ Symbol.for("tensorlake.native.close.v1");
|
|
52
|
-
|
|
53
|
-
// src/defaults.ts
|
|
54
|
-
var SDK_VERSION = "0.5.142";
|
|
55
|
-
var API_URL = process.env.TENSORLAKE_API_URL ?? "https://api.tensorlake.ai";
|
|
56
|
-
var API_KEY = process.env.TENSORLAKE_API_KEY ?? void 0;
|
|
57
|
-
var NAMESPACE = process.env.INDEXIFY_NAMESPACE ?? "default";
|
|
58
|
-
var SANDBOX_PROXY_URL = process.env.TENSORLAKE_SANDBOX_PROXY_URL ?? "https://sandbox.tensorlake.ai";
|
|
59
|
-
|
|
60
|
-
// src/url.ts
|
|
61
|
-
function trimTrailingSlashes(url) {
|
|
62
|
-
return url.endsWith("/") ? url.slice(0, -1) : url;
|
|
63
|
-
}
|
|
64
|
-
function resolveProxyTarget(proxyUrl, sandboxId) {
|
|
65
|
-
try {
|
|
66
|
-
const parsed = new URL(proxyUrl);
|
|
67
|
-
const host = parsed.hostname;
|
|
68
|
-
if (host === "localhost" || host === "127.0.0.1") {
|
|
69
|
-
return {
|
|
70
|
-
baseUrl: trimTrailingSlashes(proxyUrl),
|
|
71
|
-
hostHeader: `${sandboxId}.local`,
|
|
72
|
-
sandboxIdHeader: void 0
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
const port = parsed.port ? `:${parsed.port}` : "";
|
|
76
|
-
return {
|
|
77
|
-
baseUrl: `${parsed.protocol}//${host}${port}`,
|
|
78
|
-
hostHeader: void 0,
|
|
79
|
-
sandboxIdHeader: sandboxId
|
|
80
|
-
};
|
|
81
|
-
} catch {
|
|
82
|
-
return {
|
|
83
|
-
baseUrl: `${trimTrailingSlashes(proxyUrl)}/${sandboxId}`,
|
|
84
|
-
hostHeader: void 0,
|
|
85
|
-
sandboxIdHeader: void 0
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
// src/native-worker-client.ts
|
|
91
|
-
var NativeWorkerClient = class {
|
|
92
|
-
constructor(workerPath = () => (0, import_node_module.createRequire)(importMetaUrl).resolve(
|
|
93
|
-
"tensorlake/internal/native-worker"
|
|
94
|
-
)) {
|
|
95
|
-
this.workerPath = workerPath;
|
|
96
|
-
}
|
|
97
|
-
worker;
|
|
98
|
-
nextId = 1;
|
|
99
|
-
registered = /* @__PURE__ */ new Set();
|
|
100
|
-
pending = /* @__PURE__ */ new Map();
|
|
101
|
-
finalizers = new FinalizationRegistry(
|
|
102
|
-
(id) => this.release(id)
|
|
103
|
-
);
|
|
104
|
-
handle(kind, args, parent) {
|
|
105
|
-
const handle = { id: this.nextId++, kind, args, parent };
|
|
106
|
-
this.finalizers.register(handle, handle.id);
|
|
107
|
-
let closed = false;
|
|
108
|
-
return new Proxy(
|
|
109
|
-
{},
|
|
110
|
-
{
|
|
111
|
-
get: (_target, method) => {
|
|
112
|
-
if (method === "then") return void 0;
|
|
113
|
-
if (method === closeNativeHandle)
|
|
114
|
-
return () => {
|
|
115
|
-
closed = true;
|
|
116
|
-
this.release(handle.id);
|
|
117
|
-
};
|
|
118
|
-
if (method === "connectProxy")
|
|
119
|
-
return (...proxyArgs) => {
|
|
120
|
-
if (closed) throw new Error("Tensorlake native client is closed");
|
|
121
|
-
return this.handle("NativeSandboxProxyClient", proxyArgs, handle);
|
|
122
|
-
};
|
|
123
|
-
if (method === "baseUrl")
|
|
124
|
-
return () => resolveProxyTarget(String(args[0]), String(args[1])).baseUrl;
|
|
125
|
-
if (typeof method !== "string") return void 0;
|
|
126
|
-
return (...callArgs) => {
|
|
127
|
-
if (closed)
|
|
128
|
-
return Promise.reject(
|
|
129
|
-
new Error("Tensorlake native client is closed")
|
|
130
|
-
);
|
|
131
|
-
const emit = typeof callArgs.at(-1) === "function" ? callArgs.pop() : void 0;
|
|
132
|
-
return this.call(handle, method, callArgs, emit);
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
);
|
|
137
|
-
}
|
|
138
|
-
call(handle, method, args, emit) {
|
|
139
|
-
const id = this.nextId++;
|
|
140
|
-
let worker;
|
|
141
|
-
const promise = new Promise((resolve, reject) => {
|
|
142
|
-
try {
|
|
143
|
-
worker = this.getWorker();
|
|
144
|
-
this.pending.set(id, { resolve, reject, emit, handle });
|
|
145
|
-
worker.ref();
|
|
146
|
-
if (handle) this.register(handle, worker);
|
|
147
|
-
this.send(worker, {
|
|
148
|
-
type: "call",
|
|
149
|
-
id,
|
|
150
|
-
handleId: handle?.id,
|
|
151
|
-
method,
|
|
152
|
-
args,
|
|
153
|
-
stream: !!emit
|
|
154
|
-
});
|
|
155
|
-
} catch (error) {
|
|
156
|
-
this.pending.delete(id);
|
|
157
|
-
this.unrefIfIdle();
|
|
158
|
-
reject(error);
|
|
159
|
-
}
|
|
160
|
-
});
|
|
161
|
-
promise[cancelNativeCall] = () => {
|
|
162
|
-
const pending = this.pending.get(id);
|
|
163
|
-
if (pending && worker) {
|
|
164
|
-
pending.emit = void 0;
|
|
165
|
-
this.send(worker, { type: "cancel", id });
|
|
166
|
-
}
|
|
167
|
-
};
|
|
168
|
-
return promise;
|
|
169
|
-
}
|
|
170
|
-
getWorker() {
|
|
171
|
-
if (this.worker) return this.worker;
|
|
172
|
-
const worker = new import_node_worker_threads.Worker(this.workerPath(), {
|
|
173
|
-
// The bundled worker needs no application loaders or telemetry preloads.
|
|
174
|
-
// NODE_OPTIONS is parsed independently of execArgv by Node.
|
|
175
|
-
execArgv: [],
|
|
176
|
-
env: { ...process.env, NODE_OPTIONS: "" },
|
|
177
|
-
name: "tensorlake-native"
|
|
178
|
-
});
|
|
179
|
-
this.worker = worker;
|
|
180
|
-
worker.on("message", (message) => {
|
|
181
|
-
const pending = this.pending.get(message.id);
|
|
182
|
-
if (!pending || this.worker !== worker) return;
|
|
183
|
-
if (message.type === "event") {
|
|
184
|
-
Promise.resolve().then(() => pending.emit?.(message.value)).then(
|
|
185
|
-
() => {
|
|
186
|
-
if (this.worker === worker)
|
|
187
|
-
this.send(worker, { type: "ack", id: message.id });
|
|
188
|
-
},
|
|
189
|
-
(error) => {
|
|
190
|
-
this.send(worker, { type: "cancel", id: message.id });
|
|
191
|
-
pending.reject(error);
|
|
192
|
-
}
|
|
193
|
-
);
|
|
194
|
-
return;
|
|
195
|
-
}
|
|
196
|
-
this.pending.delete(message.id);
|
|
197
|
-
if (message.type === "error") {
|
|
198
|
-
const error = new Error(message.error.message);
|
|
199
|
-
error.name = message.error.name;
|
|
200
|
-
error.stack = message.error.stack ?? error.stack;
|
|
201
|
-
pending.reject(error);
|
|
202
|
-
} else {
|
|
203
|
-
pending.resolve(message.value);
|
|
204
|
-
}
|
|
205
|
-
this.unrefIfIdle();
|
|
206
|
-
});
|
|
207
|
-
const fail = (error) => {
|
|
208
|
-
if (this.worker !== worker) return;
|
|
209
|
-
this.worker = void 0;
|
|
210
|
-
this.registered.clear();
|
|
211
|
-
for (const pending of this.pending.values()) pending.reject(error);
|
|
212
|
-
this.pending.clear();
|
|
213
|
-
void worker.terminate();
|
|
214
|
-
};
|
|
215
|
-
worker.on("error", fail);
|
|
216
|
-
worker.on("messageerror", fail);
|
|
217
|
-
worker.on(
|
|
218
|
-
"exit",
|
|
219
|
-
(code) => fail(
|
|
220
|
-
new Error(
|
|
221
|
-
`Tensorlake native worker exited (${code}); in-flight operations were not retried`
|
|
222
|
-
)
|
|
223
|
-
)
|
|
224
|
-
);
|
|
225
|
-
worker.unref();
|
|
226
|
-
return worker;
|
|
227
|
-
}
|
|
228
|
-
register(handle, worker) {
|
|
229
|
-
if (this.registered.has(handle.id)) return;
|
|
230
|
-
if (handle.parent) this.register(handle.parent, worker);
|
|
231
|
-
this.send(worker, {
|
|
232
|
-
type: "create",
|
|
233
|
-
handle: { id: handle.id, kind: handle.kind, args: handle.args },
|
|
234
|
-
parentId: handle.parent?.id
|
|
235
|
-
});
|
|
236
|
-
this.registered.add(handle.id);
|
|
237
|
-
}
|
|
238
|
-
release(handleId) {
|
|
239
|
-
if (this.worker && this.registered.delete(handleId))
|
|
240
|
-
this.send(this.worker, { type: "release", handleId });
|
|
241
|
-
}
|
|
242
|
-
send(worker, message) {
|
|
243
|
-
worker.postMessage(message);
|
|
244
|
-
}
|
|
245
|
-
unrefIfIdle() {
|
|
246
|
-
if (this.pending.size === 0) this.worker?.unref();
|
|
247
|
-
}
|
|
248
|
-
};
|
|
249
|
-
var runtimeKey = /* @__PURE__ */ Symbol.for(
|
|
250
|
-
`tensorlake.native.worker.${false ? "source" : "0.5.142"}.v1`
|
|
251
|
-
);
|
|
252
|
-
var runtimes = globalThis;
|
|
253
|
-
function configureNativeWorker(workerPath) {
|
|
254
|
-
runtimes[runtimeKey] ??= new NativeWorkerClient(workerPath);
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
// src/function-executor/main.ts
|
|
258
|
-
var import_node_path2 = __toESM(require("path"), 1);
|
|
259
|
-
var import_node_url3 = require("url");
|
|
260
|
-
var grpc2 = __toESM(require("@grpc/grpc-js"), 1);
|
|
261
|
-
var import_proto_loader = require("@grpc/proto-loader");
|
|
262
|
-
var import_google_proto_files = require("google-proto-files");
|
|
263
|
-
|
|
264
|
-
// src/function-executor/safe-output.ts
|
|
265
|
-
var nativeJSONStringify = JSON.stringify;
|
|
266
|
-
function captureStream(stream) {
|
|
267
|
-
stream.on("error", () => void 0);
|
|
268
|
-
return {
|
|
269
|
-
// Deployed code runs in this process and can replace stream.write. Capture
|
|
270
|
-
// the executor's writer before application modules are imported.
|
|
271
|
-
write: stream.write.bind(stream)
|
|
272
|
-
};
|
|
273
|
-
}
|
|
274
|
-
var executorOutput = {
|
|
275
|
-
stdout: captureStream(process.stdout),
|
|
276
|
-
stderr: captureStream(process.stderr)
|
|
277
|
-
};
|
|
278
|
-
function writeStructuredOutput(destination, createRecord) {
|
|
279
|
-
try {
|
|
280
|
-
const serialized = nativeJSONStringify(createRecord());
|
|
281
|
-
if (serialized == null) return;
|
|
282
|
-
executorOutput[destination].write(`${serialized}
|
|
283
|
-
`, () => void 0);
|
|
284
|
-
} catch {
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
// src/function-executor/service.ts
|
|
289
|
-
var import_promises2 = require("fs/promises");
|
|
290
|
-
var import_node_os = __toESM(require("os"), 1);
|
|
291
|
-
var import_node_path = __toESM(require("path"), 1);
|
|
292
|
-
var import_node_url2 = require("url");
|
|
293
|
-
var import_node_vm = require("vm");
|
|
294
|
-
var grpc = __toESM(require("@grpc/grpc-js"), 1);
|
|
295
|
-
var import_fflate = require("fflate");
|
|
296
|
-
|
|
297
|
-
// src/applications/errors.ts
|
|
298
|
-
var SDK_USAGE_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.sdk-usage-error.v1");
|
|
299
|
-
var REQUEST_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.request-error.v1");
|
|
300
|
-
var FUNCTION_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.function-error.v1");
|
|
301
|
-
var TIMEOUT_ERROR_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.timeout-error.v1");
|
|
302
|
-
var TensorlakeApplicationError = class extends Error {
|
|
303
|
-
constructor(message, options) {
|
|
304
|
-
super(message, options);
|
|
305
|
-
this.name = "TensorlakeApplicationError";
|
|
306
|
-
}
|
|
307
|
-
};
|
|
308
|
-
var SDKUsageError = class _SDKUsageError extends TensorlakeApplicationError {
|
|
309
|
-
[SDK_USAGE_ERROR_BRAND] = true;
|
|
310
|
-
static [Symbol.hasInstance](value) {
|
|
311
|
-
if (this !== _SDKUsageError) {
|
|
312
|
-
return Function.prototype[Symbol.hasInstance].call(this, value);
|
|
313
|
-
}
|
|
314
|
-
return isSDKUsageError(value);
|
|
315
|
-
}
|
|
316
|
-
constructor(message) {
|
|
317
|
-
super(message);
|
|
318
|
-
this.name = "SDKUsageError";
|
|
319
|
-
}
|
|
320
|
-
};
|
|
321
|
-
function isSDKUsageError(value) {
|
|
322
|
-
if (typeof value !== "object" || value == null) return false;
|
|
323
|
-
const candidate = value;
|
|
324
|
-
return candidate[SDK_USAGE_ERROR_BRAND] === true && typeof candidate.message === "string";
|
|
325
|
-
}
|
|
326
|
-
var SerializationError = class extends TensorlakeApplicationError {
|
|
327
|
-
constructor(message, options) {
|
|
328
|
-
super(message, options);
|
|
329
|
-
this.name = "SerializationError";
|
|
330
|
-
}
|
|
331
|
-
};
|
|
332
|
-
var DeserializationError = class extends TensorlakeApplicationError {
|
|
333
|
-
constructor(message, options) {
|
|
334
|
-
super(message, options);
|
|
335
|
-
this.name = "DeserializationError";
|
|
336
|
-
}
|
|
337
|
-
};
|
|
338
|
-
var FunctionError = class _FunctionError extends TensorlakeApplicationError {
|
|
339
|
-
[FUNCTION_ERROR_BRAND] = true;
|
|
340
|
-
static [Symbol.hasInstance](value) {
|
|
341
|
-
if (this !== _FunctionError) {
|
|
342
|
-
return Function.prototype[Symbol.hasInstance].call(this, value);
|
|
343
|
-
}
|
|
344
|
-
return isFunctionError(value);
|
|
345
|
-
}
|
|
346
|
-
constructor(message, options) {
|
|
347
|
-
super(message, options);
|
|
348
|
-
this.name = "FunctionError";
|
|
349
|
-
}
|
|
350
|
-
};
|
|
351
|
-
var TimeoutError = class _TimeoutError extends TensorlakeApplicationError {
|
|
352
|
-
[TIMEOUT_ERROR_BRAND] = true;
|
|
353
|
-
static [Symbol.hasInstance](value) {
|
|
354
|
-
if (this !== _TimeoutError) {
|
|
355
|
-
return Function.prototype[Symbol.hasInstance].call(this, value);
|
|
356
|
-
}
|
|
357
|
-
return isTimeoutError(value);
|
|
358
|
-
}
|
|
359
|
-
constructor() {
|
|
360
|
-
super("Timed out.");
|
|
361
|
-
this.name = "TimeoutError";
|
|
362
|
-
}
|
|
363
|
-
};
|
|
364
|
-
function isTimeoutError(value) {
|
|
365
|
-
if (typeof value !== "object" || value == null) return false;
|
|
366
|
-
const candidate = value;
|
|
367
|
-
return candidate[TIMEOUT_ERROR_BRAND] === true && typeof candidate.message === "string";
|
|
368
|
-
}
|
|
369
|
-
function isFunctionError(value) {
|
|
370
|
-
if (typeof value !== "object" || value == null) return false;
|
|
371
|
-
const candidate = value;
|
|
372
|
-
return candidate[FUNCTION_ERROR_BRAND] === true && typeof candidate.message === "string";
|
|
373
|
-
}
|
|
374
|
-
var RequestError = class _RequestError extends TensorlakeApplicationError {
|
|
375
|
-
[REQUEST_ERROR_BRAND] = true;
|
|
376
|
-
static [Symbol.hasInstance](value) {
|
|
377
|
-
if (this !== _RequestError) {
|
|
378
|
-
return Function.prototype[Symbol.hasInstance].call(this, value);
|
|
379
|
-
}
|
|
380
|
-
return isRequestError(value);
|
|
381
|
-
}
|
|
382
|
-
constructor(message, options) {
|
|
383
|
-
super(message, options);
|
|
384
|
-
this.name = "RequestError";
|
|
385
|
-
}
|
|
386
|
-
};
|
|
387
|
-
function isRequestError(value) {
|
|
388
|
-
if (typeof value !== "object" || value == null) return false;
|
|
389
|
-
const candidate = value;
|
|
390
|
-
return candidate[REQUEST_ERROR_BRAND] === true && typeof candidate.message === "string";
|
|
391
|
-
}
|
|
392
|
-
var ReplayMismatchError = class extends TensorlakeApplicationError {
|
|
393
|
-
constructor(message = "Function execution diverged from its durable event history") {
|
|
394
|
-
super(message);
|
|
395
|
-
this.name = "ReplayMismatchError";
|
|
396
|
-
}
|
|
397
|
-
};
|
|
398
|
-
|
|
399
|
-
// src/applications/registry.ts
|
|
400
|
-
var REGISTRY_KEY = /* @__PURE__ */ Symbol.for("tensorlake.applications.registry.v1");
|
|
401
|
-
function registry() {
|
|
402
|
-
const target = globalThis;
|
|
403
|
-
target[REGISTRY_KEY] ??= { functions: /* @__PURE__ */ new Map(), applications: /* @__PURE__ */ new Map() };
|
|
404
|
-
return target[REGISTRY_KEY];
|
|
405
|
-
}
|
|
406
|
-
function snapshotRegistry() {
|
|
407
|
-
const state = registry();
|
|
408
|
-
return {
|
|
409
|
-
functions: new Map(state.functions),
|
|
410
|
-
applications: new Map(state.applications)
|
|
411
|
-
};
|
|
412
|
-
}
|
|
413
|
-
function restoreRegistry(snapshot) {
|
|
414
|
-
const state = registry();
|
|
415
|
-
state.functions.clear();
|
|
416
|
-
state.applications.clear();
|
|
417
|
-
for (const [name, definition] of snapshot.functions) state.functions.set(name, definition);
|
|
418
|
-
for (const [name, definition] of snapshot.applications) state.applications.set(name, definition);
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
// src/function-executor/allocation.ts
|
|
422
|
-
var import_node_async_hooks3 = require("async_hooks");
|
|
423
|
-
var import_node_crypto5 = require("crypto");
|
|
424
|
-
|
|
425
|
-
// src/applications/function.ts
|
|
426
|
-
var import_node_async_hooks = require("async_hooks");
|
|
427
|
-
var import_node_crypto = require("crypto");
|
|
428
|
-
|
|
429
|
-
// src/applications/schema.ts
|
|
430
|
-
var import_ajv = __toESM(require("ajv"), 1);
|
|
431
|
-
var import__ = __toESM(require("ajv/dist/2020.js"), 1);
|
|
432
|
-
|
|
433
|
-
// src/applications/file.ts
|
|
434
|
-
var FILE_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.file.v1");
|
|
435
|
-
var File = class _File {
|
|
436
|
-
[FILE_BRAND] = true;
|
|
437
|
-
content;
|
|
438
|
-
contentType;
|
|
439
|
-
static [Symbol.hasInstance](value) {
|
|
440
|
-
if (this !== _File) {
|
|
441
|
-
return Function.prototype[Symbol.hasInstance].call(this, value);
|
|
442
|
-
}
|
|
443
|
-
return isFile(value);
|
|
444
|
-
}
|
|
445
|
-
constructor(content, contentType) {
|
|
446
|
-
if (!contentType.trim()) {
|
|
447
|
-
throw new SDKUsageError("File contentType cannot be empty");
|
|
448
|
-
}
|
|
449
|
-
this.content = content instanceof Uint8Array ? content : new Uint8Array(content);
|
|
450
|
-
this.contentType = contentType;
|
|
451
|
-
}
|
|
452
|
-
};
|
|
453
|
-
function isFile(value) {
|
|
454
|
-
if (typeof value !== "object" || value == null) return false;
|
|
455
|
-
const candidate = value;
|
|
456
|
-
return candidate[FILE_BRAND] === true && candidate.content instanceof Uint8Array && typeof candidate.contentType === "string" && candidate.contentType.trim().length > 0;
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
// src/applications/http-body.ts
|
|
460
|
-
var HTTP_BODY_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.http-body.v1");
|
|
461
|
-
var HttpBody = class _HttpBody {
|
|
462
|
-
[HTTP_BODY_BRAND] = true;
|
|
463
|
-
content;
|
|
464
|
-
contentType;
|
|
465
|
-
static [Symbol.hasInstance](value) {
|
|
466
|
-
if (this !== _HttpBody) {
|
|
467
|
-
return Function.prototype[Symbol.hasInstance].call(this, value);
|
|
468
|
-
}
|
|
469
|
-
return isHttpBody(value);
|
|
470
|
-
}
|
|
471
|
-
constructor(content, contentType) {
|
|
472
|
-
this.content = content instanceof Uint8Array ? content : new Uint8Array(content);
|
|
473
|
-
this.contentType = contentType;
|
|
474
|
-
}
|
|
475
|
-
text(encoding = "utf-8") {
|
|
476
|
-
return new TextDecoder(encoding, { fatal: true }).decode(this.content);
|
|
477
|
-
}
|
|
478
|
-
json() {
|
|
479
|
-
return JSON.parse(this.text());
|
|
480
|
-
}
|
|
481
|
-
};
|
|
482
|
-
function isHttpBody(value) {
|
|
483
|
-
if (typeof value !== "object" || value == null) return false;
|
|
484
|
-
const candidate = value;
|
|
485
|
-
return candidate[HTTP_BODY_BRAND] === true && candidate.content instanceof Uint8Array && (candidate.contentType === void 0 || typeof candidate.contentType === "string");
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
// src/applications/schema.ts
|
|
489
|
-
var ajvDraft7 = new import_ajv.default({ allErrors: true, strict: false });
|
|
490
|
-
var ajv2020 = new import__.default({ allErrors: true, strict: false });
|
|
491
|
-
var validatorCache = /* @__PURE__ */ new WeakMap();
|
|
492
|
-
function compileValidator(jsonSchema) {
|
|
493
|
-
const dialect = jsonSchema.$schema;
|
|
494
|
-
const engine = typeof dialect === "string" && /\/draft-07\/schema#?$/.test(dialect) ? ajvDraft7 : ajv2020;
|
|
495
|
-
const validate = engine.compile(jsonSchema);
|
|
496
|
-
return {
|
|
497
|
-
validate,
|
|
498
|
-
errorsText: (errors) => engine.errorsText(errors, { separator: "; " })
|
|
499
|
-
};
|
|
500
|
-
}
|
|
501
|
-
function validateWithSchema(valueSchema, value, label) {
|
|
502
|
-
if (valueSchema._file) {
|
|
503
|
-
if (!isFile(value)) {
|
|
504
|
-
throw new SDKUsageError(`${label} must be a Tensorlake File`);
|
|
505
|
-
}
|
|
506
|
-
return value;
|
|
507
|
-
}
|
|
508
|
-
if (valueSchema._httpBody) {
|
|
509
|
-
if (!isHttpBody(value)) {
|
|
510
|
-
throw new SDKUsageError(`${label} must be a Tensorlake HttpBody`);
|
|
511
|
-
}
|
|
512
|
-
return value;
|
|
513
|
-
}
|
|
514
|
-
if (isFile(value)) {
|
|
515
|
-
throw new SDKUsageError(`${label} must use schema.file() for a Tensorlake File`);
|
|
516
|
-
}
|
|
517
|
-
if (isHttpBody(value)) {
|
|
518
|
-
throw new SDKUsageError(
|
|
519
|
-
`${label} must use schema.httpBody() for a Tensorlake HttpBody`
|
|
520
|
-
);
|
|
521
|
-
}
|
|
522
|
-
let cached = validatorCache.get(valueSchema);
|
|
523
|
-
if (cached == null) {
|
|
524
|
-
try {
|
|
525
|
-
cached = compileValidator(valueSchema.jsonSchema);
|
|
526
|
-
} catch (error) {
|
|
527
|
-
const details = error instanceof Error ? error.message : String(error);
|
|
528
|
-
throw new SDKUsageError(`${label} uses an invalid JSON schema: ${details}`);
|
|
529
|
-
}
|
|
530
|
-
validatorCache.set(valueSchema, cached);
|
|
531
|
-
}
|
|
532
|
-
if (!cached.validate(value)) {
|
|
533
|
-
const details = cached.errorsText(cached.validate.errors);
|
|
534
|
-
throw new SDKUsageError(`${label} does not match its schema: ${details}`);
|
|
535
|
-
}
|
|
536
|
-
return value;
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
// src/applications/function.ts
|
|
540
|
-
var TAIL_CALL_BRAND = /* @__PURE__ */ Symbol.for(
|
|
541
|
-
"tensorlake.applications.tail-call.v1"
|
|
542
|
-
);
|
|
543
|
-
var DURABLE_PROMISE_OBSERVER = /* @__PURE__ */ Symbol.for(
|
|
544
|
-
"tensorlake.applications.durable-promise-observer.v1"
|
|
545
|
-
);
|
|
546
|
-
var DURABLE_PROMISE_INTERNAL_THEN = /* @__PURE__ */ Symbol.for(
|
|
547
|
-
"tensorlake.applications.durable-promise-internal-then.v1"
|
|
548
|
-
);
|
|
549
|
-
var FUNCTION_RUNTIME_STORAGE_KEY = /* @__PURE__ */ Symbol.for(
|
|
550
|
-
"tensorlake.applications.function-runtime-storage.v1"
|
|
551
|
-
);
|
|
552
|
-
var PROMISE_INSTRUMENTED_KEY = /* @__PURE__ */ Symbol.for(
|
|
553
|
-
"tensorlake.applications.promise-instrumented.v3"
|
|
554
|
-
);
|
|
555
|
-
var PROMISE_THEN_INSTRUMENTED_KEY = /* @__PURE__ */ Symbol.for(
|
|
556
|
-
"tensorlake.applications.promise-then-instrumented.v1"
|
|
557
|
-
);
|
|
558
|
-
var DURABLE_THENABLE_REGISTRY_KEY = /* @__PURE__ */ Symbol.for(
|
|
559
|
-
"tensorlake.applications.durable-thenable-registry.v2"
|
|
560
|
-
);
|
|
561
|
-
var INTERNAL_PROMISE_AGGREGATE_DEPTH_KEY = /* @__PURE__ */ Symbol.for(
|
|
562
|
-
"tensorlake.applications.internal-promise-aggregate-depth.v1"
|
|
563
|
-
);
|
|
564
|
-
function runtimeStorage() {
|
|
565
|
-
const target = globalThis;
|
|
566
|
-
target[FUNCTION_RUNTIME_STORAGE_KEY] ??= new import_node_async_hooks.AsyncLocalStorage();
|
|
567
|
-
return target[FUNCTION_RUNTIME_STORAGE_KEY];
|
|
568
|
-
}
|
|
569
|
-
function currentFunctionRuntime() {
|
|
570
|
-
return runtimeStorage().getStore();
|
|
571
|
-
}
|
|
572
|
-
function runWithFunctionRuntime(runtime, callback) {
|
|
573
|
-
return runtimeStorage().run(runtime, callback);
|
|
574
|
-
}
|
|
575
|
-
function isDurableThenable(value) {
|
|
576
|
-
if (typeof value !== "object" && typeof value !== "function" || value == null) {
|
|
577
|
-
return false;
|
|
578
|
-
}
|
|
579
|
-
return durableThenableRegistry().has(value);
|
|
580
|
-
}
|
|
581
|
-
function isDetachedDurableThenable(value) {
|
|
582
|
-
const provenance = provenanceForDurableThenable(value);
|
|
583
|
-
return provenance?.detached === true || provenance?.assimilation != null && !provenance.assimilation.active;
|
|
584
|
-
}
|
|
585
|
-
function durableThenableRegistry() {
|
|
586
|
-
const target = globalThis;
|
|
587
|
-
target[DURABLE_THENABLE_REGISTRY_KEY] ??= /* @__PURE__ */ new WeakMap();
|
|
588
|
-
return target[DURABLE_THENABLE_REGISTRY_KEY];
|
|
589
|
-
}
|
|
590
|
-
function newDurableThenableMetadata() {
|
|
591
|
-
return {
|
|
592
|
-
aggregateExpectations: [],
|
|
593
|
-
provenanceExpectations: []
|
|
594
|
-
};
|
|
595
|
-
}
|
|
596
|
-
function metadataForDurableThenable(value) {
|
|
597
|
-
return provenanceForDurableThenable(value)?.metadata;
|
|
598
|
-
}
|
|
599
|
-
function provenanceForDurableThenable(value) {
|
|
600
|
-
if (typeof value !== "object" && typeof value !== "function" || value == null) {
|
|
601
|
-
return void 0;
|
|
602
|
-
}
|
|
603
|
-
return durableThenableRegistry().get(value);
|
|
604
|
-
}
|
|
605
|
-
function registerDurableThenable(value, source) {
|
|
606
|
-
const registry2 = durableThenableRegistry();
|
|
607
|
-
const metadata = source == null ? registry2.get(value)?.metadata ?? newDurableThenableMetadata() : registry2.get(source)?.metadata ?? newDurableThenableMetadata();
|
|
608
|
-
registry2.set(value, { metadata });
|
|
609
|
-
if (source != null && !registry2.has(source)) registry2.set(source, { metadata });
|
|
610
|
-
return value;
|
|
611
|
-
}
|
|
612
|
-
function registerDetachedDurableThenable(value) {
|
|
613
|
-
const registry2 = durableThenableRegistry();
|
|
614
|
-
registry2.set(value, {
|
|
615
|
-
metadata: registry2.get(value)?.metadata ?? newDurableThenableMetadata(),
|
|
616
|
-
detached: true
|
|
617
|
-
});
|
|
618
|
-
return value;
|
|
619
|
-
}
|
|
620
|
-
function inheritDurableThenable(value, source) {
|
|
621
|
-
const provenance = durableThenableRegistry().get(source);
|
|
622
|
-
if (provenance == null) return value;
|
|
623
|
-
durableThenableRegistry().set(value, provenance);
|
|
624
|
-
if (provenance.assimilation == null) {
|
|
625
|
-
provenance.metadata.provenanceExpectations.push({
|
|
626
|
-
value,
|
|
627
|
-
creatorAsyncId: (0, import_node_async_hooks.executionAsyncId)()
|
|
628
|
-
});
|
|
629
|
-
}
|
|
630
|
-
return value;
|
|
631
|
-
}
|
|
632
|
-
function registerDurableAssimilation(value, detach) {
|
|
633
|
-
const metadata = metadataForDurableThenable(value);
|
|
634
|
-
if (metadata == null) {
|
|
635
|
-
return {
|
|
636
|
-
finish: () => void 0,
|
|
637
|
-
link: (linked) => linked
|
|
638
|
-
};
|
|
639
|
-
}
|
|
640
|
-
const registration = {
|
|
641
|
-
active: true,
|
|
642
|
-
detach
|
|
643
|
-
};
|
|
644
|
-
const provenanceIndex = metadata.provenanceExpectations.findIndex(
|
|
645
|
-
(expectation) => expectation.creatorAsyncId !== (0, import_node_async_hooks.executionAsyncId)()
|
|
646
|
-
);
|
|
647
|
-
if (provenanceIndex >= 0) {
|
|
648
|
-
const [expectation] = metadata.provenanceExpectations.splice(provenanceIndex, 1);
|
|
649
|
-
durableThenableRegistry().set(expectation.value, {
|
|
650
|
-
metadata,
|
|
651
|
-
assimilation: registration
|
|
652
|
-
});
|
|
653
|
-
}
|
|
654
|
-
const expectationIndex = metadata.aggregateExpectations.findIndex(
|
|
655
|
-
(expectation) => !expectation.scope.settled && expectation.creatorAsyncId !== (0, import_node_async_hooks.executionAsyncId)()
|
|
656
|
-
);
|
|
657
|
-
if (expectationIndex >= 0) {
|
|
658
|
-
const [expectation] = metadata.aggregateExpectations.splice(expectationIndex, 1);
|
|
659
|
-
associateAssimilationWithAggregate(registration, expectation.scope);
|
|
660
|
-
}
|
|
661
|
-
return {
|
|
662
|
-
finish: () => finishDurableAssimilation(registration),
|
|
663
|
-
link: (linked) => {
|
|
664
|
-
durableThenableRegistry().set(linked, {
|
|
665
|
-
metadata,
|
|
666
|
-
assimilation: registration
|
|
667
|
-
});
|
|
668
|
-
return linked;
|
|
669
|
-
}
|
|
670
|
-
};
|
|
671
|
-
}
|
|
672
|
-
function associateAssimilationWithAggregate(registration, scope) {
|
|
673
|
-
if (!registration.active || scope.settled) return;
|
|
674
|
-
finishDurableAssimilation(registration);
|
|
675
|
-
registration.detach();
|
|
676
|
-
}
|
|
677
|
-
function finishDurableAssimilation(registration) {
|
|
678
|
-
if (!registration.active) return;
|
|
679
|
-
registration.active = false;
|
|
680
|
-
}
|
|
681
|
-
function trackDurableAggregate(runtime, result, durableContenders, markResult) {
|
|
682
|
-
const scope = {
|
|
683
|
-
settled: false,
|
|
684
|
-
metadata: /* @__PURE__ */ new Set()
|
|
685
|
-
};
|
|
686
|
-
if (markResult) {
|
|
687
|
-
registerDetachedDurableThenable(result);
|
|
688
|
-
}
|
|
689
|
-
const creatorAsyncId = (0, import_node_async_hooks.executionAsyncId)();
|
|
690
|
-
for (const contender of durableContenders) {
|
|
691
|
-
const provenance = provenanceForDurableThenable(contender);
|
|
692
|
-
if (provenance == null) continue;
|
|
693
|
-
if (provenance.assimilation != null) {
|
|
694
|
-
associateAssimilationWithAggregate(provenance.assimilation, scope);
|
|
695
|
-
} else {
|
|
696
|
-
provenance.metadata.aggregateExpectations.push({
|
|
697
|
-
scope,
|
|
698
|
-
creatorAsyncId
|
|
699
|
-
});
|
|
700
|
-
}
|
|
701
|
-
if (scope.metadata.has(provenance.metadata)) continue;
|
|
702
|
-
const { metadata } = provenance;
|
|
703
|
-
scope.metadata.add(metadata);
|
|
704
|
-
}
|
|
705
|
-
const settle = () => {
|
|
706
|
-
if (scope.settled) return;
|
|
707
|
-
scope.settled = true;
|
|
708
|
-
for (const metadata of scope.metadata) {
|
|
709
|
-
for (let index = metadata.aggregateExpectations.length - 1; index >= 0; index -= 1) {
|
|
710
|
-
if (metadata.aggregateExpectations[index].scope === scope) {
|
|
711
|
-
metadata.aggregateExpectations.splice(index, 1);
|
|
712
|
-
}
|
|
713
|
-
}
|
|
714
|
-
}
|
|
715
|
-
scope.metadata.clear();
|
|
716
|
-
};
|
|
717
|
-
if (runtime.trackPromiseSettlement?.(result, settle) !== true) settle();
|
|
718
|
-
}
|
|
719
|
-
function promiseAggregateObservation(kind) {
|
|
720
|
-
let settleLifetime;
|
|
721
|
-
const ordinaryLifetime = new Promise((resolve) => {
|
|
722
|
-
settleLifetime = resolve;
|
|
723
|
-
});
|
|
724
|
-
let contenderCount;
|
|
725
|
-
let fulfillmentCount = 0;
|
|
726
|
-
let rejectionCount = 0;
|
|
727
|
-
let settled = false;
|
|
728
|
-
const maybeSettle = () => {
|
|
729
|
-
if (settled) return;
|
|
730
|
-
const canNoLongerAdvance = kind === "any" ? contenderCount != null && rejectionCount === contenderCount : kind === "all" ? rejectionCount > 0 || contenderCount != null && fulfillmentCount === contenderCount : contenderCount != null && fulfillmentCount + rejectionCount === contenderCount;
|
|
731
|
-
if (!canNoLongerAdvance) return;
|
|
732
|
-
settled = true;
|
|
733
|
-
settleLifetime();
|
|
734
|
-
};
|
|
735
|
-
return {
|
|
736
|
-
ordinaryLifetime,
|
|
737
|
-
allOrdinaryValuesObservable: true,
|
|
738
|
-
observeFulfillment: () => {
|
|
739
|
-
fulfillmentCount += 1;
|
|
740
|
-
maybeSettle();
|
|
741
|
-
},
|
|
742
|
-
observeRejection: () => {
|
|
743
|
-
rejectionCount += 1;
|
|
744
|
-
maybeSettle();
|
|
745
|
-
},
|
|
746
|
-
finishIteration: (count) => {
|
|
747
|
-
contenderCount = count;
|
|
748
|
-
maybeSettle();
|
|
749
|
-
}
|
|
750
|
-
};
|
|
751
|
-
}
|
|
752
|
-
function observePromiseAggregateContender(normalized, observation) {
|
|
753
|
-
if (typeof normalized !== "object" && typeof normalized !== "function" || normalized == null) {
|
|
754
|
-
observation.allOrdinaryValuesObservable = false;
|
|
755
|
-
return normalized;
|
|
756
|
-
}
|
|
757
|
-
let observedSettled = false;
|
|
758
|
-
const observed = /* @__PURE__ */ Object.create(null);
|
|
759
|
-
Object.defineProperty(observed, "then", {
|
|
760
|
-
configurable: true,
|
|
761
|
-
get: () => {
|
|
762
|
-
const then = Reflect.get(normalized, "then", normalized);
|
|
763
|
-
if (typeof then !== "function") return then;
|
|
764
|
-
return function observedThen(onfulfilled, onrejected) {
|
|
765
|
-
return Reflect.apply(then, normalized, [
|
|
766
|
-
function observedFulfillment(value) {
|
|
767
|
-
if (!observedSettled) {
|
|
768
|
-
observedSettled = true;
|
|
769
|
-
observation.observeFulfillment();
|
|
770
|
-
}
|
|
771
|
-
return typeof onfulfilled === "function" ? Reflect.apply(onfulfilled, this, [value]) : void 0;
|
|
772
|
-
},
|
|
773
|
-
function observedRejection(reason) {
|
|
774
|
-
if (!observedSettled) {
|
|
775
|
-
observedSettled = true;
|
|
776
|
-
observation.observeRejection();
|
|
777
|
-
}
|
|
778
|
-
if (typeof onrejected === "function") {
|
|
779
|
-
return Reflect.apply(onrejected, this, [reason]);
|
|
780
|
-
}
|
|
781
|
-
throw reason;
|
|
782
|
-
}
|
|
783
|
-
]);
|
|
784
|
-
};
|
|
785
|
-
}
|
|
786
|
-
});
|
|
787
|
-
return observed;
|
|
788
|
-
}
|
|
789
|
-
function observedPromiseConstructor(constructor, observeNormalized) {
|
|
790
|
-
const facade = function PromiseAggregateObservedConstructor(executor) {
|
|
791
|
-
return Reflect.construct(
|
|
792
|
-
constructor,
|
|
793
|
-
[executor],
|
|
794
|
-
constructor
|
|
795
|
-
);
|
|
796
|
-
};
|
|
797
|
-
Object.defineProperty(facade, "resolve", {
|
|
798
|
-
configurable: true,
|
|
799
|
-
get: () => {
|
|
800
|
-
const promiseResolve = Reflect.get(constructor, "resolve", constructor);
|
|
801
|
-
if (typeof promiseResolve !== "function") return promiseResolve;
|
|
802
|
-
return (value) => {
|
|
803
|
-
const normalized = Reflect.apply(promiseResolve, constructor, [value]);
|
|
804
|
-
return observeNormalized(normalized);
|
|
805
|
-
};
|
|
806
|
-
}
|
|
807
|
-
});
|
|
808
|
-
return facade;
|
|
809
|
-
}
|
|
810
|
-
function isInternalPromiseAggregate() {
|
|
811
|
-
const target = globalThis;
|
|
812
|
-
return (target[INTERNAL_PROMISE_AGGREGATE_DEPTH_KEY] ?? 0) > 0;
|
|
813
|
-
}
|
|
814
|
-
function installPromiseInstrumentation() {
|
|
815
|
-
const target = globalThis;
|
|
816
|
-
if (target[PROMISE_THEN_INSTRUMENTED_KEY] !== true) {
|
|
817
|
-
const originalThen = Promise.prototype.then;
|
|
818
|
-
Object.defineProperty(Promise.prototype, "then", {
|
|
819
|
-
...Object.getOwnPropertyDescriptor(Promise.prototype, "then"),
|
|
820
|
-
value: function then(onfulfilled, onrejected) {
|
|
821
|
-
const result = Reflect.apply(
|
|
822
|
-
originalThen,
|
|
823
|
-
this,
|
|
824
|
-
[onfulfilled, onrejected]
|
|
825
|
-
);
|
|
826
|
-
if (isDetachedDurableThenable(this)) {
|
|
827
|
-
currentFunctionRuntime()?.trackDurablePromiseConsumption?.(result);
|
|
828
|
-
}
|
|
829
|
-
return result;
|
|
830
|
-
}
|
|
831
|
-
});
|
|
832
|
-
target[PROMISE_THEN_INSTRUMENTED_KEY] = true;
|
|
833
|
-
}
|
|
834
|
-
if (target[PROMISE_INSTRUMENTED_KEY] === true) return;
|
|
835
|
-
const originalRace = Promise.race;
|
|
836
|
-
const originalAny = Promise.any;
|
|
837
|
-
const originalAll = Promise.all;
|
|
838
|
-
const originalAllSettled = Promise.allSettled;
|
|
839
|
-
const originalResolve = Promise.resolve;
|
|
840
|
-
Object.defineProperty(Promise, "resolve", {
|
|
841
|
-
...Object.getOwnPropertyDescriptor(Promise, "resolve"),
|
|
842
|
-
value: function resolve(value) {
|
|
843
|
-
const result = Reflect.apply(originalResolve, this, [value]);
|
|
844
|
-
if (isDurableThenable(value)) inheritDurableThenable(result, value);
|
|
845
|
-
return result;
|
|
846
|
-
}
|
|
847
|
-
});
|
|
848
|
-
Object.defineProperty(Promise, "race", {
|
|
849
|
-
...Object.getOwnPropertyDescriptor(Promise, "race"),
|
|
850
|
-
value: function race(values) {
|
|
851
|
-
const runtime = currentFunctionRuntime();
|
|
852
|
-
if (runtime?.trackMixedPromiseRace == null) {
|
|
853
|
-
return Reflect.apply(originalRace, this, [values]);
|
|
854
|
-
}
|
|
855
|
-
const durableContenders = [];
|
|
856
|
-
let ordinaryContenderCount = 0;
|
|
857
|
-
const trackedValues = {
|
|
858
|
-
*[Symbol.iterator]() {
|
|
859
|
-
for (const value of values) {
|
|
860
|
-
yield value;
|
|
861
|
-
}
|
|
862
|
-
}
|
|
863
|
-
};
|
|
864
|
-
const observedConstructor = observedPromiseConstructor(
|
|
865
|
-
this,
|
|
866
|
-
(normalized) => {
|
|
867
|
-
if (isDurableThenable(normalized)) {
|
|
868
|
-
durableContenders.push(normalized);
|
|
869
|
-
} else {
|
|
870
|
-
ordinaryContenderCount += 1;
|
|
871
|
-
}
|
|
872
|
-
return normalized;
|
|
873
|
-
}
|
|
874
|
-
);
|
|
875
|
-
const construct = () => Reflect.apply(
|
|
876
|
-
originalRace,
|
|
877
|
-
observedConstructor,
|
|
878
|
-
[trackedValues]
|
|
879
|
-
);
|
|
880
|
-
const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
|
|
881
|
-
if (durableContenders.length > 0) {
|
|
882
|
-
trackDurableAggregate(
|
|
883
|
-
runtime,
|
|
884
|
-
result,
|
|
885
|
-
durableContenders,
|
|
886
|
-
!isInternalPromiseAggregate()
|
|
887
|
-
);
|
|
888
|
-
}
|
|
889
|
-
if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
|
|
890
|
-
runtime.trackMixedPromiseRace(result);
|
|
891
|
-
}
|
|
892
|
-
return result;
|
|
893
|
-
}
|
|
894
|
-
});
|
|
895
|
-
Object.defineProperty(Promise, "any", {
|
|
896
|
-
...Object.getOwnPropertyDescriptor(Promise, "any"),
|
|
897
|
-
value: function any(values) {
|
|
898
|
-
const runtime = currentFunctionRuntime();
|
|
899
|
-
if (runtime?.trackMixedPromiseRace == null) {
|
|
900
|
-
return Reflect.apply(originalAny, this, [values]);
|
|
901
|
-
}
|
|
902
|
-
const durableContenders = [];
|
|
903
|
-
let ordinaryContenderCount = 0;
|
|
904
|
-
const observation = promiseAggregateObservation("any");
|
|
905
|
-
const trackedValues = {
|
|
906
|
-
*[Symbol.iterator]() {
|
|
907
|
-
for (const value of values) {
|
|
908
|
-
yield value;
|
|
909
|
-
}
|
|
910
|
-
}
|
|
911
|
-
};
|
|
912
|
-
const observedConstructor = observedPromiseConstructor(
|
|
913
|
-
this,
|
|
914
|
-
(normalized) => {
|
|
915
|
-
if (isDurableThenable(normalized)) {
|
|
916
|
-
durableContenders.push(normalized);
|
|
917
|
-
return normalized;
|
|
918
|
-
}
|
|
919
|
-
ordinaryContenderCount += 1;
|
|
920
|
-
return observePromiseAggregateContender(normalized, observation);
|
|
921
|
-
}
|
|
922
|
-
);
|
|
923
|
-
const construct = () => Reflect.apply(
|
|
924
|
-
originalAny,
|
|
925
|
-
observedConstructor,
|
|
926
|
-
[trackedValues]
|
|
927
|
-
);
|
|
928
|
-
const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
|
|
929
|
-
observation.finishIteration(ordinaryContenderCount);
|
|
930
|
-
if (durableContenders.length > 0) {
|
|
931
|
-
trackDurableAggregate(
|
|
932
|
-
runtime,
|
|
933
|
-
result,
|
|
934
|
-
durableContenders,
|
|
935
|
-
!isInternalPromiseAggregate()
|
|
936
|
-
);
|
|
937
|
-
}
|
|
938
|
-
if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
|
|
939
|
-
if (observation.allOrdinaryValuesObservable) {
|
|
940
|
-
runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
|
|
941
|
-
} else {
|
|
942
|
-
runtime.trackMixedPromiseRace(result);
|
|
943
|
-
}
|
|
944
|
-
}
|
|
945
|
-
return result;
|
|
946
|
-
}
|
|
947
|
-
});
|
|
948
|
-
Object.defineProperty(Promise, "all", {
|
|
949
|
-
...Object.getOwnPropertyDescriptor(Promise, "all"),
|
|
950
|
-
value: function all(values) {
|
|
951
|
-
const runtime = currentFunctionRuntime();
|
|
952
|
-
if (runtime?.trackMixedPromiseRace == null) {
|
|
953
|
-
return Reflect.apply(originalAll, this, [values]);
|
|
954
|
-
}
|
|
955
|
-
const durableContenders = [];
|
|
956
|
-
let ordinaryContenderCount = 0;
|
|
957
|
-
const observation = promiseAggregateObservation("all");
|
|
958
|
-
const trackedValues = {
|
|
959
|
-
*[Symbol.iterator]() {
|
|
960
|
-
for (const value of values) {
|
|
961
|
-
yield value;
|
|
962
|
-
}
|
|
963
|
-
}
|
|
964
|
-
};
|
|
965
|
-
const observedConstructor = observedPromiseConstructor(
|
|
966
|
-
this,
|
|
967
|
-
(normalized) => {
|
|
968
|
-
if (isDurableThenable(normalized)) {
|
|
969
|
-
durableContenders.push(normalized);
|
|
970
|
-
return normalized;
|
|
971
|
-
}
|
|
972
|
-
ordinaryContenderCount += 1;
|
|
973
|
-
return observePromiseAggregateContender(normalized, observation);
|
|
974
|
-
}
|
|
975
|
-
);
|
|
976
|
-
const construct = () => Reflect.apply(
|
|
977
|
-
originalAll,
|
|
978
|
-
observedConstructor,
|
|
979
|
-
[trackedValues]
|
|
980
|
-
);
|
|
981
|
-
const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
|
|
982
|
-
observation.finishIteration(ordinaryContenderCount);
|
|
983
|
-
if (durableContenders.length > 0) {
|
|
984
|
-
trackDurableAggregate(
|
|
985
|
-
runtime,
|
|
986
|
-
result,
|
|
987
|
-
durableContenders,
|
|
988
|
-
!isInternalPromiseAggregate()
|
|
989
|
-
);
|
|
990
|
-
}
|
|
991
|
-
if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
|
|
992
|
-
if (observation.allOrdinaryValuesObservable) {
|
|
993
|
-
runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
|
|
994
|
-
} else {
|
|
995
|
-
runtime.trackMixedPromiseRace(result);
|
|
996
|
-
}
|
|
997
|
-
}
|
|
998
|
-
return result;
|
|
999
|
-
}
|
|
1000
|
-
});
|
|
1001
|
-
Object.defineProperty(Promise, "allSettled", {
|
|
1002
|
-
...Object.getOwnPropertyDescriptor(Promise, "allSettled"),
|
|
1003
|
-
value: function allSettled(values) {
|
|
1004
|
-
const runtime = currentFunctionRuntime();
|
|
1005
|
-
if (runtime?.trackMixedPromiseRace == null) {
|
|
1006
|
-
return Reflect.apply(
|
|
1007
|
-
originalAllSettled,
|
|
1008
|
-
this,
|
|
1009
|
-
[values]
|
|
1010
|
-
);
|
|
1011
|
-
}
|
|
1012
|
-
const durableContenders = [];
|
|
1013
|
-
let ordinaryContenderCount = 0;
|
|
1014
|
-
const observation = promiseAggregateObservation("all_settled");
|
|
1015
|
-
const trackedValues = {
|
|
1016
|
-
*[Symbol.iterator]() {
|
|
1017
|
-
for (const value of values) yield value;
|
|
1018
|
-
}
|
|
1019
|
-
};
|
|
1020
|
-
const observedConstructor = observedPromiseConstructor(
|
|
1021
|
-
this,
|
|
1022
|
-
(normalized) => {
|
|
1023
|
-
if (isDurableThenable(normalized)) {
|
|
1024
|
-
durableContenders.push(normalized);
|
|
1025
|
-
return normalized;
|
|
1026
|
-
}
|
|
1027
|
-
ordinaryContenderCount += 1;
|
|
1028
|
-
return observePromiseAggregateContender(normalized, observation);
|
|
1029
|
-
}
|
|
1030
|
-
);
|
|
1031
|
-
const construct = () => Reflect.apply(
|
|
1032
|
-
originalAllSettled,
|
|
1033
|
-
observedConstructor,
|
|
1034
|
-
[trackedValues]
|
|
1035
|
-
);
|
|
1036
|
-
const result = runtime.runPromiseAggregateConstruction?.(construct) ?? construct();
|
|
1037
|
-
observation.finishIteration(ordinaryContenderCount);
|
|
1038
|
-
if (durableContenders.length > 0) {
|
|
1039
|
-
trackDurableAggregate(
|
|
1040
|
-
runtime,
|
|
1041
|
-
result,
|
|
1042
|
-
durableContenders,
|
|
1043
|
-
!isInternalPromiseAggregate()
|
|
1044
|
-
);
|
|
1045
|
-
}
|
|
1046
|
-
if (durableContenders.length > 0 && ordinaryContenderCount > 0) {
|
|
1047
|
-
if (observation.allOrdinaryValuesObservable) {
|
|
1048
|
-
runtime.trackMixedPromiseRace(result, [observation.ordinaryLifetime, result]);
|
|
1049
|
-
} else {
|
|
1050
|
-
runtime.trackMixedPromiseRace(result);
|
|
1051
|
-
}
|
|
1052
|
-
}
|
|
1053
|
-
return result;
|
|
1054
|
-
}
|
|
1055
|
-
});
|
|
1056
|
-
target[PROMISE_INSTRUMENTED_KEY] = true;
|
|
1057
|
-
}
|
|
1058
|
-
function validateArguments(definition, supplied) {
|
|
1059
|
-
const args = [...supplied];
|
|
1060
|
-
if (args.length > definition.parameters.length) {
|
|
1061
|
-
throw new SDKUsageError(`${definition.name} expected at most ${definition.parameters.length} arguments, received ${args.length}`);
|
|
1062
|
-
}
|
|
1063
|
-
definition.parameters.forEach((parameter, index) => {
|
|
1064
|
-
if (index >= args.length || args[index] === void 0) {
|
|
1065
|
-
if (parameter.hasDefault) args[index] = parameter.defaultValue;
|
|
1066
|
-
else if (parameter.required) throw new SDKUsageError(`${definition.name} is missing argument '${parameter.name}'`);
|
|
1067
|
-
return;
|
|
1068
|
-
}
|
|
1069
|
-
validateWithSchema(parameter.schema, args[index], `${definition.name}.${parameter.name}`);
|
|
1070
|
-
});
|
|
1071
|
-
return args;
|
|
1072
|
-
}
|
|
1073
|
-
async function executeHandlerResult(definition, suppliedArgs) {
|
|
1074
|
-
const args = validateArguments(definition, suppliedArgs);
|
|
1075
|
-
const result = definition.handler(...args);
|
|
1076
|
-
if (result == null || typeof result.then !== "function") {
|
|
1077
|
-
throw new SDKUsageError(`Tensorlake handler '${definition.name}' must be async and return a Promise`);
|
|
1078
|
-
}
|
|
1079
|
-
const output = await result;
|
|
1080
|
-
if (isTailCall(output)) {
|
|
1081
|
-
return output;
|
|
1082
|
-
}
|
|
1083
|
-
validateWithSchema(definition.returns, output, `${definition.name} return value`);
|
|
1084
|
-
return output;
|
|
1085
|
-
}
|
|
1086
|
-
function isTailCall(value) {
|
|
1087
|
-
if (typeof value !== "object" || value == null) return false;
|
|
1088
|
-
const candidate = value;
|
|
1089
|
-
return candidate[TAIL_CALL_BRAND] === true && candidate.kind === "tensorlake-tail-call" && typeof candidate.definition === "object" && candidate.definition != null && Array.isArray(candidate.args);
|
|
1090
|
-
}
|
|
1091
|
-
|
|
1092
|
-
// src/applications/headers.ts
|
|
1093
|
-
var HEADERS_BRAND = /* @__PURE__ */ Symbol.for("tensorlake.applications.headers.v1");
|
|
1094
|
-
var Headers2 = class _Headers {
|
|
1095
|
-
[HEADERS_BRAND] = true;
|
|
1096
|
-
#items;
|
|
1097
|
-
#lookup;
|
|
1098
|
-
static [Symbol.hasInstance](value) {
|
|
1099
|
-
if (this !== _Headers) {
|
|
1100
|
-
return Function.prototype[Symbol.hasInstance].call(this, value);
|
|
1101
|
-
}
|
|
1102
|
-
return isHeaders(value);
|
|
1103
|
-
}
|
|
1104
|
-
constructor(headers = []) {
|
|
1105
|
-
const entries = headers instanceof _Headers ? [...headers] : Symbol.iterator in Object(headers) ? [...headers] : Object.entries(headers);
|
|
1106
|
-
const lookup = /* @__PURE__ */ new Map();
|
|
1107
|
-
this.#items = Object.freeze(entries.map(([name, value]) => {
|
|
1108
|
-
if (typeof name !== "string" || typeof value !== "string") {
|
|
1109
|
-
throw new TypeError("Headers require string names and values");
|
|
1110
|
-
}
|
|
1111
|
-
const item = Object.freeze([name, value]);
|
|
1112
|
-
const key = name.toLowerCase();
|
|
1113
|
-
lookup.set(key, [...lookup.get(key) ?? [], value]);
|
|
1114
|
-
return item;
|
|
1115
|
-
}));
|
|
1116
|
-
this.#lookup = lookup;
|
|
1117
|
-
Object.freeze(this);
|
|
1118
|
-
}
|
|
1119
|
-
get(name) {
|
|
1120
|
-
return this.#lookup.get(name.toLowerCase())?.at(-1);
|
|
1121
|
-
}
|
|
1122
|
-
getAll(name) {
|
|
1123
|
-
return Object.freeze([...this.#lookup.get(name.toLowerCase()) ?? []]);
|
|
1124
|
-
}
|
|
1125
|
-
has(name) {
|
|
1126
|
-
return this.#lookup.has(name.toLowerCase());
|
|
1127
|
-
}
|
|
1128
|
-
entries() {
|
|
1129
|
-
return this.#items[Symbol.iterator]();
|
|
1130
|
-
}
|
|
1131
|
-
[Symbol.iterator]() {
|
|
1132
|
-
return this.entries();
|
|
1133
|
-
}
|
|
1134
|
-
};
|
|
1135
|
-
function isHeaders(value) {
|
|
1136
|
-
if (typeof value !== "object" || value == null) return false;
|
|
1137
|
-
const candidate = value;
|
|
1138
|
-
return candidate[HEADERS_BRAND] === true && typeof candidate.get === "function" && typeof candidate.getAll === "function" && typeof candidate.has === "function" && typeof candidate[Symbol.iterator] === "function";
|
|
1139
|
-
}
|
|
1140
|
-
|
|
1141
|
-
// src/applications/context.ts
|
|
1142
|
-
var import_node_async_hooks2 = require("async_hooks");
|
|
1143
|
-
|
|
1144
|
-
// src/applications/serialization.ts
|
|
1145
|
-
var import_node_crypto2 = require("crypto");
|
|
1146
|
-
var textEncoder = new TextEncoder();
|
|
1147
|
-
var textDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
1148
|
-
function assertJSONValue(value, path3, seen) {
|
|
1149
|
-
if (value === null || typeof value === "string" || typeof value === "boolean") return;
|
|
1150
|
-
if (typeof value === "number") {
|
|
1151
|
-
if (!Number.isFinite(value)) throw new SerializationError(`${path3} contains a non-finite number`);
|
|
1152
|
-
return;
|
|
1153
|
-
}
|
|
1154
|
-
if (typeof value !== "object") {
|
|
1155
|
-
throw new SerializationError(`${path3} contains unsupported ${typeof value}`);
|
|
1156
|
-
}
|
|
1157
|
-
if (isFile(value)) {
|
|
1158
|
-
throw new SerializationError(`${path3} contains a nested File; File is only supported as a direct value`);
|
|
1159
|
-
}
|
|
1160
|
-
if (isHttpBody(value)) {
|
|
1161
|
-
throw new SerializationError(
|
|
1162
|
-
`${path3} contains a nested HttpBody; HttpBody is only supported as a direct value`
|
|
1163
|
-
);
|
|
1164
|
-
}
|
|
1165
|
-
if (seen.has(value)) throw new SerializationError(`${path3} contains a cycle`);
|
|
1166
|
-
if (Object.getPrototypeOf(value) !== Object.prototype && !Array.isArray(value)) {
|
|
1167
|
-
throw new SerializationError(`${path3} contains unsupported ${value.constructor?.name ?? "object"}`);
|
|
1168
|
-
}
|
|
1169
|
-
seen.add(value);
|
|
1170
|
-
if (Array.isArray(value)) {
|
|
1171
|
-
for (let index = 0; index < value.length; index += 1) {
|
|
1172
|
-
if (!Object.prototype.hasOwnProperty.call(value, index)) {
|
|
1173
|
-
throw new SerializationError(`${path3} contains a sparse array entry at index ${index}`);
|
|
1174
|
-
}
|
|
1175
|
-
assertJSONValue(value[index], `${path3}[${index}]`, seen);
|
|
1176
|
-
}
|
|
1177
|
-
} else {
|
|
1178
|
-
for (const [key, item] of Object.entries(value)) {
|
|
1179
|
-
assertJSONValue(item, `${path3}.${key}`, seen);
|
|
1180
|
-
}
|
|
1181
|
-
}
|
|
1182
|
-
seen.delete(value);
|
|
1183
|
-
}
|
|
1184
|
-
function serializeValue(value) {
|
|
1185
|
-
if (isFile(value)) {
|
|
1186
|
-
return { data: value.content, contentType: value.contentType, encoding: "raw" };
|
|
1187
|
-
}
|
|
1188
|
-
if (isHttpBody(value)) {
|
|
1189
|
-
return {
|
|
1190
|
-
data: value.content,
|
|
1191
|
-
contentType: value.contentType ?? "",
|
|
1192
|
-
encoding: "raw"
|
|
1193
|
-
};
|
|
1194
|
-
}
|
|
1195
|
-
try {
|
|
1196
|
-
assertJSONValue(value, "value", /* @__PURE__ */ new Set());
|
|
1197
|
-
return {
|
|
1198
|
-
data: textEncoder.encode(JSON.stringify(value)),
|
|
1199
|
-
contentType: "application/json; charset=UTF-8",
|
|
1200
|
-
encoding: "json"
|
|
1201
|
-
};
|
|
1202
|
-
} catch (error) {
|
|
1203
|
-
if (error instanceof SerializationError) throw error;
|
|
1204
|
-
throw new SerializationError("Failed to serialize JSON value", { cause: error });
|
|
1205
|
-
}
|
|
1206
|
-
}
|
|
1207
|
-
function deserializeJSON(data) {
|
|
1208
|
-
try {
|
|
1209
|
-
return JSON.parse(textDecoder.decode(data));
|
|
1210
|
-
} catch (error) {
|
|
1211
|
-
throw new DeserializationError("Failed to deserialize JSON value", { cause: error });
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
|
|
1215
|
-
// src/applications/context.ts
|
|
1216
|
-
function validateRequestStateKey(key) {
|
|
1217
|
-
if (typeof key !== "string") {
|
|
1218
|
-
throw new SDKUsageError(`State key must be a string, got: ${String(key)}`);
|
|
1219
|
-
}
|
|
1220
|
-
}
|
|
1221
|
-
function serializeRequestStateValue(value) {
|
|
1222
|
-
const serialized = serializeValue(value);
|
|
1223
|
-
if (serialized.encoding !== "json") {
|
|
1224
|
-
throw new SDKUsageError("Request state values must be JSON values");
|
|
1225
|
-
}
|
|
1226
|
-
return serialized.data;
|
|
1227
|
-
}
|
|
1228
|
-
function validateCounterMetric(name, value) {
|
|
1229
|
-
if (typeof name !== "string") {
|
|
1230
|
-
throw new SDKUsageError(`Counter name must be a string, got: ${String(name)}`);
|
|
1231
|
-
}
|
|
1232
|
-
if (typeof value !== "number" || !Number.isInteger(value)) {
|
|
1233
|
-
throw new SDKUsageError(`Counter value must be an int, got: ${String(value)}`);
|
|
1234
|
-
}
|
|
1235
|
-
if (!Number.isSafeInteger(value)) {
|
|
1236
|
-
throw new SDKUsageError(`Counter value must be a safe int, got: ${String(value)}`);
|
|
1237
|
-
}
|
|
1238
|
-
}
|
|
1239
|
-
function validateTimerMetric(name, value) {
|
|
1240
|
-
if (typeof name !== "string") {
|
|
1241
|
-
throw new SDKUsageError(`Timer name must be a string, got: ${String(name)}`);
|
|
1242
|
-
}
|
|
1243
|
-
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
1244
|
-
throw new SDKUsageError(`Timer value must be a finite number, got: ${String(value)}`);
|
|
1245
|
-
}
|
|
1246
|
-
}
|
|
1247
|
-
function isPlainRecord(value) {
|
|
1248
|
-
if (value == null || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1249
|
-
const prototype = Object.getPrototypeOf(value);
|
|
1250
|
-
return prototype === Object.prototype || prototype === null;
|
|
1251
|
-
}
|
|
1252
|
-
function validateProgressUpdate(current, total, options) {
|
|
1253
|
-
if (!Number.isFinite(current) || !Number.isFinite(total) || total < 0 || current < 0) {
|
|
1254
|
-
throw new SDKUsageError("Progress current and total must be non-negative finite numbers");
|
|
1255
|
-
}
|
|
1256
|
-
if (options != null && !isPlainRecord(options)) {
|
|
1257
|
-
throw new SDKUsageError("Progress options must be an object");
|
|
1258
|
-
}
|
|
1259
|
-
if (options?.message != null && typeof options.message !== "string") {
|
|
1260
|
-
throw new SDKUsageError("Progress message must be a string");
|
|
1261
|
-
}
|
|
1262
|
-
if (options?.attributes != null) {
|
|
1263
|
-
if (!isPlainRecord(options.attributes)) {
|
|
1264
|
-
throw new SDKUsageError("Progress attributes must be an object of string key/value pairs");
|
|
1265
|
-
}
|
|
1266
|
-
for (const key of Reflect.ownKeys(options.attributes)) {
|
|
1267
|
-
if (typeof key !== "string") {
|
|
1268
|
-
throw new SDKUsageError("Progress attributes must contain only string keys and values");
|
|
1269
|
-
}
|
|
1270
|
-
const value = options.attributes[key];
|
|
1271
|
-
if (typeof value !== "string") {
|
|
1272
|
-
throw new SDKUsageError("Progress attributes must contain only string keys and values");
|
|
1273
|
-
}
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
}
|
|
1277
|
-
var REQUEST_CONTEXT_STORAGE_KEY = /* @__PURE__ */ Symbol.for(
|
|
1278
|
-
"tensorlake.applications.request-context-storage.v1"
|
|
1279
|
-
);
|
|
1280
|
-
function requestContextStorage() {
|
|
1281
|
-
const target = globalThis;
|
|
1282
|
-
target[REQUEST_CONTEXT_STORAGE_KEY] ??= new import_node_async_hooks2.AsyncLocalStorage();
|
|
1283
|
-
return target[REQUEST_CONTEXT_STORAGE_KEY];
|
|
1284
|
-
}
|
|
1285
|
-
function runWithRequestContext(context, callback) {
|
|
1286
|
-
return requestContextStorage().run(context, callback);
|
|
1287
|
-
}
|
|
1288
|
-
async function waitWithAbortSignal(value, signal) {
|
|
1289
|
-
if (signal.aborted) throw signal.reason;
|
|
1290
|
-
let onAbort;
|
|
1291
|
-
try {
|
|
1292
|
-
return await Promise.race([
|
|
1293
|
-
value,
|
|
1294
|
-
new Promise((_resolve, reject) => {
|
|
1295
|
-
onAbort = () => reject(signal.reason);
|
|
1296
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
1297
|
-
})
|
|
1298
|
-
]);
|
|
1299
|
-
} finally {
|
|
1300
|
-
if (onAbort != null) signal.removeEventListener("abort", onAbort);
|
|
1301
|
-
}
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
|
-
// src/function-executor/async-queue.ts
|
|
1305
|
-
function deferred() {
|
|
1306
|
-
let resolve;
|
|
1307
|
-
let reject;
|
|
1308
|
-
const promise = new Promise((ok, fail) => {
|
|
1309
|
-
resolve = ok;
|
|
1310
|
-
reject = fail;
|
|
1311
|
-
});
|
|
1312
|
-
return { promise, resolve, reject };
|
|
1313
|
-
}
|
|
1314
|
-
|
|
1315
|
-
// src/function-executor/blob.ts
|
|
1316
|
-
var import_node_crypto3 = require("crypto");
|
|
1317
|
-
var import_promises = require("fs/promises");
|
|
1318
|
-
var import_node_url = require("url");
|
|
1319
|
-
function enumIs(value, name, numberValue) {
|
|
1320
|
-
return value === name || value === numberValue;
|
|
1321
|
-
}
|
|
1322
|
-
function concatenate(parts) {
|
|
1323
|
-
const result = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
|
|
1324
|
-
let offset = 0;
|
|
1325
|
-
for (const part of parts) {
|
|
1326
|
-
result.set(part, offset);
|
|
1327
|
-
offset += part.byteLength;
|
|
1328
|
-
}
|
|
1329
|
-
return result;
|
|
1330
|
-
}
|
|
1331
|
-
function networkURI(uri) {
|
|
1332
|
-
if (uri.startsWith("s3://")) return `https://${uri.slice("s3://".length)}`;
|
|
1333
|
-
if (uri.startsWith("gs://")) return `https://${uri.slice("gs://".length)}`;
|
|
1334
|
-
return uri;
|
|
1335
|
-
}
|
|
1336
|
-
var BlobHTTPError = class extends Error {
|
|
1337
|
-
constructor(status2) {
|
|
1338
|
-
super(`BLOB operation failed with HTTP ${status2}`);
|
|
1339
|
-
this.status = status2;
|
|
1340
|
-
}
|
|
1341
|
-
};
|
|
1342
|
-
function uriKind(uri) {
|
|
1343
|
-
if (uri.startsWith("file://")) return "file";
|
|
1344
|
-
if (uri.startsWith("s3://")) return "s3";
|
|
1345
|
-
if (uri.startsWith("gs://")) return "gcs";
|
|
1346
|
-
try {
|
|
1347
|
-
return new URL(uri).protocol.replace(":", "") || "unknown";
|
|
1348
|
-
} catch {
|
|
1349
|
-
return "unknown";
|
|
1350
|
-
}
|
|
1351
|
-
}
|
|
1352
|
-
function throwIfAborted(signal) {
|
|
1353
|
-
signal?.throwIfAborted();
|
|
1354
|
-
}
|
|
1355
|
-
async function abortableDelay(delayMs, signal) {
|
|
1356
|
-
throwIfAborted(signal);
|
|
1357
|
-
if (signal == null) {
|
|
1358
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
1359
|
-
return;
|
|
1360
|
-
}
|
|
1361
|
-
await new Promise((resolve, reject) => {
|
|
1362
|
-
const timer = setTimeout(() => {
|
|
1363
|
-
signal.removeEventListener("abort", onAbort);
|
|
1364
|
-
resolve();
|
|
1365
|
-
}, delayMs);
|
|
1366
|
-
const onAbort = () => {
|
|
1367
|
-
clearTimeout(timer);
|
|
1368
|
-
reject(signal.reason);
|
|
1369
|
-
};
|
|
1370
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
1371
|
-
});
|
|
1372
|
-
}
|
|
1373
|
-
async function fetchWithRetries(uri, init, log, fields = {}, signal) {
|
|
1374
|
-
let lastError;
|
|
1375
|
-
for (let attempt = 0; attempt < 4; attempt += 1) {
|
|
1376
|
-
throwIfAborted(signal);
|
|
1377
|
-
const startedAt = Date.now();
|
|
1378
|
-
log?.("HTTP attempt starting", {
|
|
1379
|
-
...fields,
|
|
1380
|
-
attempt: attempt + 1,
|
|
1381
|
-
method: init?.method ?? "GET",
|
|
1382
|
-
uri_kind: uriKind(uri),
|
|
1383
|
-
has_range: new Headers(init?.headers).has("range")
|
|
1384
|
-
});
|
|
1385
|
-
try {
|
|
1386
|
-
const response = await fetch(networkURI(uri), {
|
|
1387
|
-
...init,
|
|
1388
|
-
signal
|
|
1389
|
-
});
|
|
1390
|
-
log?.("HTTP attempt completed", {
|
|
1391
|
-
...fields,
|
|
1392
|
-
attempt: attempt + 1,
|
|
1393
|
-
method: init?.method ?? "GET",
|
|
1394
|
-
status_code: response.status,
|
|
1395
|
-
duration_ms: Date.now() - startedAt
|
|
1396
|
-
});
|
|
1397
|
-
throwIfAborted(signal);
|
|
1398
|
-
if (response.ok) return response;
|
|
1399
|
-
if ([400, 403, 404].includes(response.status) || attempt === 3) {
|
|
1400
|
-
throw new BlobHTTPError(response.status);
|
|
1401
|
-
}
|
|
1402
|
-
lastError = new BlobHTTPError(response.status);
|
|
1403
|
-
} catch (error) {
|
|
1404
|
-
log?.("HTTP attempt failed", {
|
|
1405
|
-
...fields,
|
|
1406
|
-
attempt: attempt + 1,
|
|
1407
|
-
method: init?.method ?? "GET",
|
|
1408
|
-
duration_ms: Date.now() - startedAt,
|
|
1409
|
-
error: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
|
|
1410
|
-
});
|
|
1411
|
-
throwIfAborted(signal);
|
|
1412
|
-
if (error instanceof BlobHTTPError && [400, 403, 404].includes(error.status)) throw error;
|
|
1413
|
-
lastError = error;
|
|
1414
|
-
if (attempt === 3) break;
|
|
1415
|
-
}
|
|
1416
|
-
const retryDelayMs = 100 * 2 ** attempt;
|
|
1417
|
-
log?.("HTTP attempt scheduled for retry", {
|
|
1418
|
-
...fields,
|
|
1419
|
-
next_attempt: attempt + 2,
|
|
1420
|
-
retry_delay_ms: retryDelayMs
|
|
1421
|
-
});
|
|
1422
|
-
await abortableDelay(retryDelayMs, signal);
|
|
1423
|
-
}
|
|
1424
|
-
throw new Error("BLOB operation failed after retries", { cause: lastError });
|
|
1425
|
-
}
|
|
1426
|
-
async function readChunkRange(chunk, index, logicalChunkOffset, offsetWithinChunk, size, sharedURI, log, signal) {
|
|
1427
|
-
throwIfAborted(signal);
|
|
1428
|
-
if (!chunk.uri) throw new Error("BLOB chunk has no URI");
|
|
1429
|
-
const physicalOffset = sharedURI ? logicalChunkOffset + offsetWithinChunk : offsetWithinChunk;
|
|
1430
|
-
const fields = {
|
|
1431
|
-
chunk_index: index,
|
|
1432
|
-
chunk_offset: logicalChunkOffset,
|
|
1433
|
-
read_offset: physicalOffset,
|
|
1434
|
-
read_bytes: size,
|
|
1435
|
-
uri_kind: uriKind(chunk.uri),
|
|
1436
|
-
shared_uri: sharedURI
|
|
1437
|
-
};
|
|
1438
|
-
const startedAt = Date.now();
|
|
1439
|
-
log?.("download chunk starting", fields);
|
|
1440
|
-
if (chunk.uri.startsWith("file://")) {
|
|
1441
|
-
const file = await (0, import_promises.open)((0, import_node_url.fileURLToPath)(chunk.uri), "r");
|
|
1442
|
-
try {
|
|
1443
|
-
throwIfAborted(signal);
|
|
1444
|
-
const data2 = new Uint8Array(size);
|
|
1445
|
-
const { bytesRead } = await file.read(data2, 0, size, physicalOffset);
|
|
1446
|
-
throwIfAborted(signal);
|
|
1447
|
-
if (bytesRead !== size) {
|
|
1448
|
-
throw new Error(`BLOB chunk ended after ${bytesRead} bytes; expected ${size}`);
|
|
1449
|
-
}
|
|
1450
|
-
log?.("download chunk completed", {
|
|
1451
|
-
...fields,
|
|
1452
|
-
downloaded_bytes: data2.byteLength,
|
|
1453
|
-
duration_ms: Date.now() - startedAt
|
|
1454
|
-
});
|
|
1455
|
-
return data2;
|
|
1456
|
-
} finally {
|
|
1457
|
-
await file.close();
|
|
1458
|
-
}
|
|
1459
|
-
}
|
|
1460
|
-
const headers = { range: `bytes=${physicalOffset}-${physicalOffset + size - 1}` };
|
|
1461
|
-
const response = await fetchWithRetries(chunk.uri, { headers }, log, fields, signal);
|
|
1462
|
-
let data = new Uint8Array(await response.arrayBuffer());
|
|
1463
|
-
throwIfAborted(signal);
|
|
1464
|
-
if (response.status === 200 && (physicalOffset !== 0 || data.byteLength !== size)) {
|
|
1465
|
-
data = data.subarray(physicalOffset, physicalOffset + size);
|
|
1466
|
-
}
|
|
1467
|
-
if (data.byteLength !== size) {
|
|
1468
|
-
throw new Error(`BLOB chunk returned ${data.byteLength} bytes; expected ${size}`);
|
|
1469
|
-
}
|
|
1470
|
-
log?.("download chunk completed", {
|
|
1471
|
-
...fields,
|
|
1472
|
-
status_code: response.status,
|
|
1473
|
-
downloaded_bytes: data.byteLength,
|
|
1474
|
-
duration_ms: Date.now() - startedAt
|
|
1475
|
-
});
|
|
1476
|
-
return data;
|
|
1477
|
-
}
|
|
1478
|
-
async function downloadBlobRange(blob, offset, size, log, signal) {
|
|
1479
|
-
throwIfAborted(signal);
|
|
1480
|
-
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(size) || size < 0) {
|
|
1481
|
-
throw new Error("BLOB range offset and size must be non-negative safe integers");
|
|
1482
|
-
}
|
|
1483
|
-
const chunks = blob.chunks ?? [];
|
|
1484
|
-
const totalSize = chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0);
|
|
1485
|
-
if (offset + size > totalSize) {
|
|
1486
|
-
throw new Error(`BLOB range ${offset}..${offset + size} exceeds declared size ${totalSize}`);
|
|
1487
|
-
}
|
|
1488
|
-
if (size === 0) return new Uint8Array();
|
|
1489
|
-
const uriCounts = /* @__PURE__ */ new Map();
|
|
1490
|
-
for (const chunk of chunks) {
|
|
1491
|
-
if (chunk.uri != null) uriCounts.set(chunk.uri, (uriCounts.get(chunk.uri) ?? 0) + 1);
|
|
1492
|
-
}
|
|
1493
|
-
const rangeEnd = offset + size;
|
|
1494
|
-
let chunkOffset = 0;
|
|
1495
|
-
const reads = [];
|
|
1496
|
-
for (const [index, chunk] of chunks.entries()) {
|
|
1497
|
-
const chunkSize = chunk.size ?? 0;
|
|
1498
|
-
const chunkEnd = chunkOffset + chunkSize;
|
|
1499
|
-
const overlapStart = Math.max(offset, chunkOffset);
|
|
1500
|
-
const overlapEnd = Math.min(rangeEnd, chunkEnd);
|
|
1501
|
-
if (overlapStart < overlapEnd) {
|
|
1502
|
-
reads.push(readChunkRange(
|
|
1503
|
-
chunk,
|
|
1504
|
-
index,
|
|
1505
|
-
chunkOffset,
|
|
1506
|
-
overlapStart - chunkOffset,
|
|
1507
|
-
overlapEnd - overlapStart,
|
|
1508
|
-
(uriCounts.get(chunk.uri ?? "") ?? 0) > 1,
|
|
1509
|
-
log,
|
|
1510
|
-
signal
|
|
1511
|
-
));
|
|
1512
|
-
}
|
|
1513
|
-
chunkOffset = chunkEnd;
|
|
1514
|
-
}
|
|
1515
|
-
const data = concatenate(await Promise.all(reads));
|
|
1516
|
-
throwIfAborted(signal);
|
|
1517
|
-
if (data.byteLength !== size) throw new Error("BLOB does not contain the complete requested range");
|
|
1518
|
-
log?.("download blob completed", {
|
|
1519
|
-
blob_id: blob.id,
|
|
1520
|
-
chunk_count: chunks.length,
|
|
1521
|
-
downloaded_bytes: data.byteLength,
|
|
1522
|
-
range_offset: offset,
|
|
1523
|
-
range_size: size
|
|
1524
|
-
});
|
|
1525
|
-
return data;
|
|
1526
|
-
}
|
|
1527
|
-
async function downloadBlob(blob, log, signal) {
|
|
1528
|
-
const size = (blob.chunks ?? []).reduce((total, chunk) => total + (chunk.size ?? 0), 0);
|
|
1529
|
-
return downloadBlobRange(blob, 0, size, log, signal);
|
|
1530
|
-
}
|
|
1531
|
-
async function downloadSerializedObject(object, blob, log, signal) {
|
|
1532
|
-
throwIfAborted(signal);
|
|
1533
|
-
const manifest = object.manifest;
|
|
1534
|
-
if (manifest?.size == null || manifest.metadataSize == null) {
|
|
1535
|
-
throw new Error("Serialized object manifest is missing size or metadata_size");
|
|
1536
|
-
}
|
|
1537
|
-
if (!Number.isSafeInteger(manifest.size) || manifest.size < 0) {
|
|
1538
|
-
throw new Error("Serialized object size must be a non-negative safe integer");
|
|
1539
|
-
}
|
|
1540
|
-
if (!Number.isSafeInteger(manifest.metadataSize) || manifest.metadataSize < 0) {
|
|
1541
|
-
throw new Error("Serialized object metadata_size must be a non-negative safe integer");
|
|
1542
|
-
}
|
|
1543
|
-
if (manifest.metadataSize > manifest.size) {
|
|
1544
|
-
throw new Error("Serialized object metadata_size cannot exceed size");
|
|
1545
|
-
}
|
|
1546
|
-
if (typeof manifest.sha256Hash !== "string" || manifest.sha256Hash.length === 0) {
|
|
1547
|
-
throw new Error("Serialized object manifest is missing sha256_hash");
|
|
1548
|
-
}
|
|
1549
|
-
const start = object.offset ?? 0;
|
|
1550
|
-
const bytes = await downloadBlobRange(blob, start, manifest.size, log, signal);
|
|
1551
|
-
throwIfAborted(signal);
|
|
1552
|
-
const hash = (0, import_node_crypto3.createHash)("sha256").update(bytes).digest("hex");
|
|
1553
|
-
if (hash !== manifest.sha256Hash) {
|
|
1554
|
-
throw new Error(`Serialized object hash mismatch: expected ${manifest.sha256Hash}, got ${hash}`);
|
|
1555
|
-
}
|
|
1556
|
-
return {
|
|
1557
|
-
metadata: bytes.subarray(0, manifest.metadataSize),
|
|
1558
|
-
data: bytes.subarray(manifest.metadataSize),
|
|
1559
|
-
contentType: manifest.contentType,
|
|
1560
|
-
encoding: manifest.encoding
|
|
1561
|
-
};
|
|
1562
|
-
}
|
|
1563
|
-
function deserializeValueFromProtocol(value) {
|
|
1564
|
-
if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_RAW", 5)) {
|
|
1565
|
-
return new File(value.data, value.contentType ?? "application/octet-stream");
|
|
1566
|
-
}
|
|
1567
|
-
if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_UTF8_TEXT", 2)) {
|
|
1568
|
-
return new TextDecoder("utf-8", { fatal: true }).decode(value.data);
|
|
1569
|
-
}
|
|
1570
|
-
if (enumIs(value.encoding, "SERIALIZED_OBJECT_ENCODING_UTF8_JSON", 1) || value.contentType?.toLowerCase().includes("json")) {
|
|
1571
|
-
return deserializeJSON(value.data);
|
|
1572
|
-
}
|
|
1573
|
-
throw new Error(`Unsupported serialized object encoding '${String(value.encoding)}'`);
|
|
1574
|
-
}
|
|
1575
|
-
function prepareSerializedObject(value, offset = 0, sourceFunctionCallId) {
|
|
1576
|
-
const serialized = serializeValue(value);
|
|
1577
|
-
const metadata = new TextEncoder().encode(JSON.stringify({
|
|
1578
|
-
format: "tensorlake.typescript.value.v1"
|
|
1579
|
-
}));
|
|
1580
|
-
const bytes = concatenate([metadata, serialized.data]);
|
|
1581
|
-
return {
|
|
1582
|
-
bytes,
|
|
1583
|
-
object: {
|
|
1584
|
-
offset,
|
|
1585
|
-
manifest: {
|
|
1586
|
-
encoding: serialized.encoding === "json" ? "SERIALIZED_OBJECT_ENCODING_UTF8_JSON" : "SERIALIZED_OBJECT_ENCODING_RAW",
|
|
1587
|
-
encodingVersion: 0,
|
|
1588
|
-
size: bytes.byteLength,
|
|
1589
|
-
metadataSize: metadata.byteLength,
|
|
1590
|
-
sha256Hash: (0, import_node_crypto3.createHash)("sha256").update(bytes).digest("hex"),
|
|
1591
|
-
...serialized.encoding === "raw" ? { contentType: serialized.contentType } : {},
|
|
1592
|
-
sourceFunctionCallId
|
|
1593
|
-
}
|
|
1594
|
-
}
|
|
1595
|
-
};
|
|
1596
|
-
}
|
|
1597
|
-
function prepareTextObject(value) {
|
|
1598
|
-
const bytes = new TextEncoder().encode(value);
|
|
1599
|
-
return {
|
|
1600
|
-
bytes,
|
|
1601
|
-
object: {
|
|
1602
|
-
offset: 0,
|
|
1603
|
-
manifest: {
|
|
1604
|
-
encoding: "SERIALIZED_OBJECT_ENCODING_UTF8_TEXT",
|
|
1605
|
-
encodingVersion: 0,
|
|
1606
|
-
size: bytes.byteLength,
|
|
1607
|
-
metadataSize: 0,
|
|
1608
|
-
sha256Hash: (0, import_node_crypto3.createHash)("sha256").update(bytes).digest("hex")
|
|
1609
|
-
}
|
|
1610
|
-
}
|
|
1611
|
-
};
|
|
1612
|
-
}
|
|
1613
|
-
async function uploadChunk(chunk, data, index, offset, log, signal) {
|
|
1614
|
-
throwIfAborted(signal);
|
|
1615
|
-
if (!chunk.uri) throw new Error("Writable BLOB chunk has no URI");
|
|
1616
|
-
const fields = {
|
|
1617
|
-
chunk_index: index,
|
|
1618
|
-
chunk_offset: offset,
|
|
1619
|
-
upload_bytes: data.byteLength,
|
|
1620
|
-
chunk_capacity: chunk.size ?? 0,
|
|
1621
|
-
uri_kind: uriKind(chunk.uri)
|
|
1622
|
-
};
|
|
1623
|
-
const startedAt = Date.now();
|
|
1624
|
-
log?.("upload chunk starting", fields);
|
|
1625
|
-
if (chunk.uri.startsWith("file://")) {
|
|
1626
|
-
const filePath = (0, import_node_url.fileURLToPath)(chunk.uri);
|
|
1627
|
-
let file;
|
|
1628
|
-
try {
|
|
1629
|
-
file = await (0, import_promises.open)(filePath, "r+");
|
|
1630
|
-
} catch (error) {
|
|
1631
|
-
if (error.code !== "ENOENT") throw error;
|
|
1632
|
-
file = await (0, import_promises.open)(filePath, "w+");
|
|
1633
|
-
}
|
|
1634
|
-
try {
|
|
1635
|
-
throwIfAborted(signal);
|
|
1636
|
-
await file.write(data, 0, data.byteLength, offset);
|
|
1637
|
-
throwIfAborted(signal);
|
|
1638
|
-
} finally {
|
|
1639
|
-
await file.close();
|
|
1640
|
-
}
|
|
1641
|
-
log?.("upload chunk completed", { ...fields, duration_ms: Date.now() - startedAt });
|
|
1642
|
-
return { ...chunk, size: data.byteLength };
|
|
1643
|
-
}
|
|
1644
|
-
const response = await fetchWithRetries(chunk.uri, {
|
|
1645
|
-
method: "PUT",
|
|
1646
|
-
...data.byteLength === 0 ? {} : {
|
|
1647
|
-
body: Uint8Array.from(data).buffer
|
|
1648
|
-
}
|
|
1649
|
-
}, log, fields, signal);
|
|
1650
|
-
throwIfAborted(signal);
|
|
1651
|
-
const uploaded = {
|
|
1652
|
-
...chunk,
|
|
1653
|
-
size: data.byteLength,
|
|
1654
|
-
etag: response.headers.get("etag") ?? void 0
|
|
1655
|
-
};
|
|
1656
|
-
log?.("upload chunk completed", {
|
|
1657
|
-
...fields,
|
|
1658
|
-
status_code: response.status,
|
|
1659
|
-
has_etag: uploaded.etag != null,
|
|
1660
|
-
duration_ms: Date.now() - startedAt
|
|
1661
|
-
});
|
|
1662
|
-
return uploaded;
|
|
1663
|
-
}
|
|
1664
|
-
async function uploadBlob(blob, data, log, signal) {
|
|
1665
|
-
throwIfAborted(signal);
|
|
1666
|
-
const chunks = blob.chunks ?? [];
|
|
1667
|
-
const capacity = chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0);
|
|
1668
|
-
if (data.byteLength > capacity) throw new Error(`BLOB capacity ${capacity} is smaller than ${data.byteLength}`);
|
|
1669
|
-
let offset = 0;
|
|
1670
|
-
const uploaded = [];
|
|
1671
|
-
for (const [index, chunk] of chunks.entries()) {
|
|
1672
|
-
const size = Math.min(chunk.size ?? 0, data.byteLength - offset);
|
|
1673
|
-
if (size <= 0) break;
|
|
1674
|
-
uploaded.push(await uploadChunk(
|
|
1675
|
-
chunk,
|
|
1676
|
-
data.subarray(offset, offset + size),
|
|
1677
|
-
index,
|
|
1678
|
-
offset,
|
|
1679
|
-
log,
|
|
1680
|
-
signal
|
|
1681
|
-
));
|
|
1682
|
-
offset += size;
|
|
1683
|
-
}
|
|
1684
|
-
if (data.byteLength === 0 && chunks.length > 0) {
|
|
1685
|
-
uploaded.push(await uploadChunk(chunks[0], data, 0, 0, log, signal));
|
|
1686
|
-
}
|
|
1687
|
-
throwIfAborted(signal);
|
|
1688
|
-
if (offset !== data.byteLength) throw new Error(`Only uploaded ${offset} of ${data.byteLength} bytes`);
|
|
1689
|
-
log?.("upload blob completed", {
|
|
1690
|
-
blob_id: blob.id,
|
|
1691
|
-
chunk_count: uploaded.length,
|
|
1692
|
-
uploaded_bytes: data.byteLength
|
|
1693
|
-
});
|
|
1694
|
-
return { ...blob, chunks: uploaded };
|
|
1695
|
-
}
|
|
1696
|
-
function joinPrepared(values) {
|
|
1697
|
-
return concatenate(values.map((value) => value.bytes));
|
|
1698
|
-
}
|
|
1699
|
-
|
|
1700
|
-
// src/function-executor/user-events.ts
|
|
1701
|
-
var import_node_crypto4 = require("crypto");
|
|
1702
|
-
function printCloudEvent(data, options = {}) {
|
|
1703
|
-
writeStructuredOutput("stdout", () => ({
|
|
1704
|
-
specversion: "1.0",
|
|
1705
|
-
id: (0, import_node_crypto4.randomUUID)(),
|
|
1706
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1707
|
-
type: options.type ?? "ai.tensorlake.event",
|
|
1708
|
-
source: options.source ?? "/tensorlake/function_executor/events",
|
|
1709
|
-
data,
|
|
1710
|
-
...options.message == null ? {} : { message: options.message }
|
|
1711
|
-
}));
|
|
1712
|
-
}
|
|
1713
|
-
|
|
1714
|
-
// src/function-executor/allocation.ts
|
|
1715
|
-
installPromiseInstrumentation();
|
|
1716
|
-
var OK = 0;
|
|
1717
|
-
var NOT_FOUND = 5;
|
|
1718
|
-
var REPLAY_STRICT = /* @__PURE__ */ new Set(["REPLAY_MODE_STRICT", 1]);
|
|
1719
|
-
var OUTCOME_SUCCESS = /* @__PURE__ */ new Set(["ALLOCATION_OUTCOME_CODE_SUCCESS", 1]);
|
|
1720
|
-
var OUTCOME_FAILURE = /* @__PURE__ */ new Set(["ALLOCATION_OUTCOME_CODE_FAILURE", 2]);
|
|
1721
|
-
var WATCHER_TIMED_OUT = /* @__PURE__ */ new Set(["FUNCTION_CALL_WATCHER_STATUS_TIMEDOUT", 2]);
|
|
1722
|
-
var MAX_REDUCE_CALLS_PER_PLAN = 512;
|
|
1723
|
-
var nativeStructuredClone = globalThis.structuredClone;
|
|
1724
|
-
function statusError(status2, fallback) {
|
|
1725
|
-
if (status2 == null || (status2.code ?? OK) === OK) return void 0;
|
|
1726
|
-
return new FunctionError(status2.message || fallback);
|
|
1727
|
-
}
|
|
1728
|
-
function validatedEventLogClock(response, requestedAfterClock, createError) {
|
|
1729
|
-
if (response.entries != null && !Array.isArray(response.entries)) {
|
|
1730
|
-
throw createError("Allocation event log returned a non-array entries field");
|
|
1731
|
-
}
|
|
1732
|
-
const responseClock = Number(response.lastClock ?? requestedAfterClock);
|
|
1733
|
-
if (!Number.isFinite(responseClock) || !Number.isSafeInteger(responseClock) || responseClock < 0) {
|
|
1734
|
-
throw createError(
|
|
1735
|
-
`Allocation event log returned invalid clock '${String(response.lastClock)}'`
|
|
1736
|
-
);
|
|
1737
|
-
}
|
|
1738
|
-
if (responseClock < requestedAfterClock) {
|
|
1739
|
-
throw createError(
|
|
1740
|
-
`Allocation event log moved backwards from clock ${requestedAfterClock} to ${responseClock}`
|
|
1741
|
-
);
|
|
1742
|
-
}
|
|
1743
|
-
if (response.hasMore && responseClock === requestedAfterClock) {
|
|
1744
|
-
throw createError(
|
|
1745
|
-
`Allocation event log reported more entries without advancing clock ${requestedAfterClock}`
|
|
1746
|
-
);
|
|
1747
|
-
}
|
|
1748
|
-
if ((response.entries?.length ?? 0) > 0 && responseClock === requestedAfterClock) {
|
|
1749
|
-
throw createError(
|
|
1750
|
-
`Allocation event log returned entries without advancing clock ${requestedAfterClock}`
|
|
1751
|
-
);
|
|
1752
|
-
}
|
|
1753
|
-
const eventFields = [
|
|
1754
|
-
"functionCallCreated",
|
|
1755
|
-
"functionCallWatcherCreated",
|
|
1756
|
-
"functionCallWatcherResult"
|
|
1757
|
-
];
|
|
1758
|
-
let previousEntryClock = requestedAfterClock;
|
|
1759
|
-
for (const [index, entry] of (response.entries ?? []).entries()) {
|
|
1760
|
-
if (entry == null || typeof entry !== "object") {
|
|
1761
|
-
throw createError(`Allocation event log entry ${index} is not an object`);
|
|
1762
|
-
}
|
|
1763
|
-
const presentFields = eventFields.filter((field) => entry[field] != null);
|
|
1764
|
-
if (presentFields.length !== 1) {
|
|
1765
|
-
throw createError(
|
|
1766
|
-
`Allocation event log entry ${index} has ${presentFields.length} recognized payloads; expected exactly one`
|
|
1767
|
-
);
|
|
1768
|
-
}
|
|
1769
|
-
const entryClock = Number(entry.clock);
|
|
1770
|
-
if (!Number.isSafeInteger(entryClock) || entryClock <= previousEntryClock || entryClock > responseClock) {
|
|
1771
|
-
throw createError(
|
|
1772
|
-
`Allocation event log entry ${index} has invalid clock '${String(entry.clock)}' after ${previousEntryClock} with page clock ${responseClock}`
|
|
1773
|
-
);
|
|
1774
|
-
}
|
|
1775
|
-
previousEntryClock = entryClock;
|
|
1776
|
-
}
|
|
1777
|
-
return responseClock;
|
|
1778
|
-
}
|
|
1779
|
-
function durableHash(values) {
|
|
1780
|
-
const hash = (0, import_node_crypto5.createHash)("sha256");
|
|
1781
|
-
for (const value of values) {
|
|
1782
|
-
const bytes = Buffer.from(value, "utf8");
|
|
1783
|
-
const length = Buffer.alloc(8);
|
|
1784
|
-
length.writeBigUInt64BE(BigInt(bytes.byteLength));
|
|
1785
|
-
hash.update(length);
|
|
1786
|
-
hash.update(bytes);
|
|
1787
|
-
hash.update("|");
|
|
1788
|
-
}
|
|
1789
|
-
return hash.digest("hex");
|
|
1790
|
-
}
|
|
1791
|
-
function protocolTimestamp(date) {
|
|
1792
|
-
const milliseconds = date.getTime();
|
|
1793
|
-
return { seconds: Math.floor(milliseconds / 1e3), nanos: milliseconds % 1e3 * 1e6 };
|
|
1794
|
-
}
|
|
1795
|
-
function parseHTTPMessage(data) {
|
|
1796
|
-
const marker = Buffer.from("\r\n\r\n");
|
|
1797
|
-
const bytes = Buffer.from(data);
|
|
1798
|
-
const split = bytes.indexOf(marker);
|
|
1799
|
-
if (split < 0) throw new DeserializationError("Invalid message/http application input");
|
|
1800
|
-
const lines = bytes.subarray(0, split).toString("utf8").split("\r\n");
|
|
1801
|
-
lines.shift();
|
|
1802
|
-
const headers = {};
|
|
1803
|
-
for (const line of lines) {
|
|
1804
|
-
const colon = line.indexOf(":");
|
|
1805
|
-
if (colon > 0) headers[line.slice(0, colon).trim().toLowerCase()] = line.slice(colon + 1).trim();
|
|
1806
|
-
}
|
|
1807
|
-
return { headers, body: new Uint8Array(bytes.subarray(split + marker.byteLength)) };
|
|
1808
|
-
}
|
|
1809
|
-
async function deserializeApplicationArguments(definition, payload) {
|
|
1810
|
-
let data = payload.data;
|
|
1811
|
-
let suppliedContentType = payload.contentType;
|
|
1812
|
-
let hasExplicitContentType = payload.contentType != null;
|
|
1813
|
-
if (suppliedContentType?.toLowerCase().startsWith("message/http")) {
|
|
1814
|
-
const parsed = parseHTTPMessage(data);
|
|
1815
|
-
data = parsed.body;
|
|
1816
|
-
const bodyContentType = parsed.headers["content-type"];
|
|
1817
|
-
hasExplicitContentType = bodyContentType != null;
|
|
1818
|
-
suppliedContentType = bodyContentType;
|
|
1819
|
-
}
|
|
1820
|
-
if (definition.parameters.length === 0) return { args: [] };
|
|
1821
|
-
const firstParameter = definition.parameters[0];
|
|
1822
|
-
if (definition.parameters.length === 1 && firstParameter?.schema._httpBody) {
|
|
1823
|
-
return { args: [new HttpBody(data, suppliedContentType)] };
|
|
1824
|
-
}
|
|
1825
|
-
const contentType = suppliedContentType ?? "application/json";
|
|
1826
|
-
if (contentType.toLowerCase().startsWith("multipart/form-data")) {
|
|
1827
|
-
let form;
|
|
1828
|
-
try {
|
|
1829
|
-
form = await new Response(Uint8Array.from(data).buffer, {
|
|
1830
|
-
headers: { "content-type": contentType }
|
|
1831
|
-
}).formData();
|
|
1832
|
-
} catch (error) {
|
|
1833
|
-
throw new DeserializationError("Failed to deserialize multipart application input", { cause: error });
|
|
1834
|
-
}
|
|
1835
|
-
const args = [];
|
|
1836
|
-
for (const [index, parameter] of definition.parameters.entries()) {
|
|
1837
|
-
const part = form.get(parameter.name) ?? form.get(String(index));
|
|
1838
|
-
if (part == null) {
|
|
1839
|
-
args.push(void 0);
|
|
1840
|
-
} else if (parameter.schema._file || parameter.schema._httpBody) {
|
|
1841
|
-
const bytes = typeof part === "string" ? new TextEncoder().encode(part) : new Uint8Array(await part.arrayBuffer());
|
|
1842
|
-
const partContentType = typeof part === "string" ? "application/octet-stream" : part.type || "application/octet-stream";
|
|
1843
|
-
args.push(parameter.schema._httpBody ? new HttpBody(bytes, partContentType) : new File(bytes, partContentType));
|
|
1844
|
-
} else {
|
|
1845
|
-
try {
|
|
1846
|
-
const bytes = typeof part === "string" ? new TextEncoder().encode(part) : new Uint8Array(await part.arrayBuffer());
|
|
1847
|
-
args.push(deserializeJSON(bytes));
|
|
1848
|
-
} catch (error) {
|
|
1849
|
-
if (error instanceof DeserializationError) throw error;
|
|
1850
|
-
throw new DeserializationError(
|
|
1851
|
-
`Failed to deserialize multipart argument '${parameter.name}' as JSON`,
|
|
1852
|
-
{ cause: error }
|
|
1853
|
-
);
|
|
1854
|
-
}
|
|
1855
|
-
}
|
|
1856
|
-
}
|
|
1857
|
-
return { args };
|
|
1858
|
-
}
|
|
1859
|
-
if (data.byteLength === 0 && (!firstParameter?.schema._file || !hasExplicitContentType)) {
|
|
1860
|
-
return { args: [] };
|
|
1861
|
-
}
|
|
1862
|
-
if (firstParameter?.schema._file) return { args: [new File(data, contentType)] };
|
|
1863
|
-
if (!contentType.toLowerCase().includes("json")) {
|
|
1864
|
-
throw new DeserializationError(
|
|
1865
|
-
`Expected application/json input for '${firstParameter?.name ?? "argument"}', got ${contentType}`
|
|
1866
|
-
);
|
|
1867
|
-
}
|
|
1868
|
-
return { args: [deserializeJSON(data)] };
|
|
1869
|
-
}
|
|
1870
|
-
var PASSIVE_ASYNC_RESOURCE_TYPES = /* @__PURE__ */ new Set([
|
|
1871
|
-
"PIPEWRAP",
|
|
1872
|
-
"PIPESERVERWRAP",
|
|
1873
|
-
"SIGNALWRAP",
|
|
1874
|
-
"TCPSERVERWRAP",
|
|
1875
|
-
"TCPWRAP",
|
|
1876
|
-
"TLSWRAP",
|
|
1877
|
-
"TTYWRAP",
|
|
1878
|
-
"UDPWRAP"
|
|
1879
|
-
]);
|
|
1880
|
-
var ReplayCausality = class {
|
|
1881
|
-
constructor(onQuiescent) {
|
|
1882
|
-
this.onQuiescent = onQuiescent;
|
|
1883
|
-
this.hook = (0, import_node_async_hooks3.createHook)({
|
|
1884
|
-
init: (asyncId, type, triggerAsyncId, resource) => {
|
|
1885
|
-
if (this.stopped) return;
|
|
1886
|
-
const triggerPromise = this.resources.get(triggerAsyncId)?.promise?.deref();
|
|
1887
|
-
const resumesDetachedDurable = type === "PROMISE" && triggerPromise != null && isDetachedDurableThenable(triggerPromise);
|
|
1888
|
-
const children = this.children.get(triggerAsyncId) ?? /* @__PURE__ */ new Set();
|
|
1889
|
-
children.add(asyncId);
|
|
1890
|
-
this.children.set(triggerAsyncId, children);
|
|
1891
|
-
if (type === "PROMISE" && typeof resource === "object" && resource != null) {
|
|
1892
|
-
this.promiseIds.set(resource, asyncId);
|
|
1893
|
-
}
|
|
1894
|
-
const triggerOwners = new Set(this.owners.get(triggerAsyncId) ?? []);
|
|
1895
|
-
this.resources.set(asyncId, {
|
|
1896
|
-
type,
|
|
1897
|
-
triggerAsyncId,
|
|
1898
|
-
promise: type === "PROMISE" && typeof resource === "object" && resource != null ? new WeakRef(resource) : void 0,
|
|
1899
|
-
pending: true
|
|
1900
|
-
});
|
|
1901
|
-
if (resumesDetachedDurable) {
|
|
1902
|
-
this.nativeDurableAssimilations.add(asyncId);
|
|
1903
|
-
this.promiseAggregateAssimilationScopes.at(-1)?.add(asyncId);
|
|
1904
|
-
this.suspendDurableAssimilation(asyncId);
|
|
1905
|
-
}
|
|
1906
|
-
if (this.suppressOwnership) return;
|
|
1907
|
-
const inherited = /* @__PURE__ */ new Set([
|
|
1908
|
-
...this.owners.get((0, import_node_async_hooks3.executionAsyncId)()) ?? [],
|
|
1909
|
-
...triggerOwners
|
|
1910
|
-
]);
|
|
1911
|
-
for (const token of inherited) this.addTokenToSubtree(asyncId, token);
|
|
1912
|
-
},
|
|
1913
|
-
promiseResolve: (asyncId) => {
|
|
1914
|
-
if (this.stopped) return;
|
|
1915
|
-
this.notifyPromiseSettlement(asyncId);
|
|
1916
|
-
if (this.nativeDurableAssimilations.delete(asyncId)) {
|
|
1917
|
-
this.resumeDurableAssimilation(asyncId);
|
|
1918
|
-
}
|
|
1919
|
-
if (this.explicitDurableAssimilations.delete(asyncId)) {
|
|
1920
|
-
this.resumeDurableAssimilation(asyncId);
|
|
1921
|
-
}
|
|
1922
|
-
this.settleMixedPromiseProducerGroups(asyncId);
|
|
1923
|
-
const currentOwners = this.owners.get((0, import_node_async_hooks3.executionAsyncId)());
|
|
1924
|
-
if (currentOwners != null) {
|
|
1925
|
-
for (const token of currentOwners) this.addTokenToSubtree(asyncId, token);
|
|
1926
|
-
}
|
|
1927
|
-
this.setResourcePending(asyncId, false);
|
|
1928
|
-
this.scheduleCheck();
|
|
1929
|
-
},
|
|
1930
|
-
destroy: (asyncId) => {
|
|
1931
|
-
if (this.stopped) return;
|
|
1932
|
-
this.notifyPromiseSettlement(asyncId);
|
|
1933
|
-
if (this.nativeDurableAssimilations.delete(asyncId)) {
|
|
1934
|
-
this.detachDurableAssimilation(asyncId);
|
|
1935
|
-
}
|
|
1936
|
-
if (this.explicitDurableAssimilations.delete(asyncId)) {
|
|
1937
|
-
this.detachDurableAssimilation(asyncId);
|
|
1938
|
-
}
|
|
1939
|
-
this.settleMixedPromiseProducerGroups(asyncId);
|
|
1940
|
-
this.setResourcePending(asyncId, false);
|
|
1941
|
-
this.releaseDestroyedResource(asyncId);
|
|
1942
|
-
this.scheduleCheck();
|
|
1943
|
-
}
|
|
1944
|
-
});
|
|
1945
|
-
this.hook.enable();
|
|
1946
|
-
}
|
|
1947
|
-
hook;
|
|
1948
|
-
children = /* @__PURE__ */ new Map();
|
|
1949
|
-
resources = /* @__PURE__ */ new Map();
|
|
1950
|
-
owners = /* @__PURE__ */ new Map();
|
|
1951
|
-
provenance = /* @__PURE__ */ new Map();
|
|
1952
|
-
tokenResources = /* @__PURE__ */ new Map();
|
|
1953
|
-
tokenProducers = /* @__PURE__ */ new Map();
|
|
1954
|
-
promiseIds = /* @__PURE__ */ new WeakMap();
|
|
1955
|
-
excludedRoots = /* @__PURE__ */ new Set();
|
|
1956
|
-
durableAssimilations = /* @__PURE__ */ new Map();
|
|
1957
|
-
nativeDurableAssimilations = /* @__PURE__ */ new Set();
|
|
1958
|
-
explicitDurableAssimilations = /* @__PURE__ */ new Set();
|
|
1959
|
-
promiseAggregateAssimilationScopes = [];
|
|
1960
|
-
promiseSettlementObservers = /* @__PURE__ */ new Map();
|
|
1961
|
-
mixedPromiseProducerGroups = /* @__PURE__ */ new Set();
|
|
1962
|
-
nextToken = 1;
|
|
1963
|
-
checkScheduled = false;
|
|
1964
|
-
checkHandle;
|
|
1965
|
-
suppressOwnership = false;
|
|
1966
|
-
stopped = false;
|
|
1967
|
-
runRoot(callback) {
|
|
1968
|
-
const resource = new import_node_async_hooks3.AsyncResource("TensorlakeReplayCausality");
|
|
1969
|
-
let result;
|
|
1970
|
-
resource.runInAsyncScope(() => {
|
|
1971
|
-
this.beginCurrentContinuation();
|
|
1972
|
-
result = callback();
|
|
1973
|
-
this.excludePromise(result);
|
|
1974
|
-
});
|
|
1975
|
-
resource.emitDestroy();
|
|
1976
|
-
return result;
|
|
1977
|
-
}
|
|
1978
|
-
beginCurrentContinuation() {
|
|
1979
|
-
if (this.stopped) return;
|
|
1980
|
-
const asyncId = (0, import_node_async_hooks3.executionAsyncId)();
|
|
1981
|
-
this.excludedRoots.delete(asyncId);
|
|
1982
|
-
this.addTokenToSubtree(asyncId, this.nextToken);
|
|
1983
|
-
this.nextToken += 1;
|
|
1984
|
-
this.scheduleCheck();
|
|
1985
|
-
}
|
|
1986
|
-
excludePromise(promise) {
|
|
1987
|
-
const asyncId = this.promiseIds.get(promise);
|
|
1988
|
-
if (asyncId == null) return;
|
|
1989
|
-
this.excludedRoots.add(asyncId);
|
|
1990
|
-
this.removeOwnersFromSubtree(asyncId);
|
|
1991
|
-
this.scheduleCheck();
|
|
1992
|
-
}
|
|
1993
|
-
suspendDurableAssimilation(asyncId) {
|
|
1994
|
-
if (this.stopped) return false;
|
|
1995
|
-
const count = this.durableAssimilations.get(asyncId) ?? 0;
|
|
1996
|
-
this.durableAssimilations.set(asyncId, count + 1);
|
|
1997
|
-
if (count === 0) {
|
|
1998
|
-
this.excludedRoots.add(asyncId);
|
|
1999
|
-
this.removeOwnersFromSubtree(asyncId);
|
|
2000
|
-
}
|
|
2001
|
-
this.scheduleCheck();
|
|
2002
|
-
return true;
|
|
2003
|
-
}
|
|
2004
|
-
resumeDurableAssimilation(asyncId) {
|
|
2005
|
-
if (this.stopped) return;
|
|
2006
|
-
const count = this.durableAssimilations.get(asyncId);
|
|
2007
|
-
if (count == null) return;
|
|
2008
|
-
if (count > 1) {
|
|
2009
|
-
this.durableAssimilations.set(asyncId, count - 1);
|
|
2010
|
-
return;
|
|
2011
|
-
}
|
|
2012
|
-
this.durableAssimilations.delete(asyncId);
|
|
2013
|
-
this.excludedRoots.delete(asyncId);
|
|
2014
|
-
const currentOwners = this.owners.get((0, import_node_async_hooks3.executionAsyncId)());
|
|
2015
|
-
if (currentOwners != null) {
|
|
2016
|
-
for (const token of currentOwners) this.addTokenToSubtree(asyncId, token);
|
|
2017
|
-
}
|
|
2018
|
-
}
|
|
2019
|
-
detachDurableAssimilation(asyncId) {
|
|
2020
|
-
if (this.stopped) return;
|
|
2021
|
-
const count = this.durableAssimilations.get(asyncId);
|
|
2022
|
-
if (count == null) return;
|
|
2023
|
-
if (count > 1) {
|
|
2024
|
-
this.durableAssimilations.set(asyncId, count - 1);
|
|
2025
|
-
return;
|
|
2026
|
-
}
|
|
2027
|
-
this.durableAssimilations.delete(asyncId);
|
|
2028
|
-
this.excludedRoots.delete(asyncId);
|
|
2029
|
-
this.scheduleCheck();
|
|
2030
|
-
}
|
|
2031
|
-
hasPendingDurableAssimilation() {
|
|
2032
|
-
return this.durableAssimilations.size > 0;
|
|
2033
|
-
}
|
|
2034
|
-
hasPendingProducer() {
|
|
2035
|
-
if (this.mixedPromiseProducerGroups.size > 0) return true;
|
|
2036
|
-
for (const producers of this.tokenProducers.values()) {
|
|
2037
|
-
if (producers.size > 0) return true;
|
|
2038
|
-
}
|
|
2039
|
-
return false;
|
|
2040
|
-
}
|
|
2041
|
-
requestCheck() {
|
|
2042
|
-
this.scheduleCheck();
|
|
2043
|
-
}
|
|
2044
|
-
trackMixedPromiseRace(promise, producers = [promise]) {
|
|
2045
|
-
if (this.stopped) return;
|
|
2046
|
-
const producerIds = /* @__PURE__ */ new Set();
|
|
2047
|
-
for (const producer of producers) {
|
|
2048
|
-
const asyncId = this.promiseIds.get(producer);
|
|
2049
|
-
if (asyncId == null) continue;
|
|
2050
|
-
if (this.resources.get(asyncId)?.pending !== true) {
|
|
2051
|
-
this.scheduleCheck();
|
|
2052
|
-
return;
|
|
2053
|
-
}
|
|
2054
|
-
producerIds.add(asyncId);
|
|
2055
|
-
}
|
|
2056
|
-
if (producerIds.size > 0) this.mixedPromiseProducerGroups.add(producerIds);
|
|
2057
|
-
this.scheduleCheck();
|
|
2058
|
-
}
|
|
2059
|
-
trackPromiseSettlement(promise, onSettled) {
|
|
2060
|
-
if (this.stopped) return false;
|
|
2061
|
-
const asyncId = this.promiseIds.get(promise);
|
|
2062
|
-
if (asyncId == null) return false;
|
|
2063
|
-
if (this.resources.get(asyncId)?.pending !== true) {
|
|
2064
|
-
onSettled();
|
|
2065
|
-
return true;
|
|
2066
|
-
}
|
|
2067
|
-
const observers = this.promiseSettlementObservers.get(asyncId) ?? /* @__PURE__ */ new Set();
|
|
2068
|
-
observers.add(onSettled);
|
|
2069
|
-
this.promiseSettlementObservers.set(asyncId, observers);
|
|
2070
|
-
return true;
|
|
2071
|
-
}
|
|
2072
|
-
trackDurablePromiseConsumption(promise) {
|
|
2073
|
-
if (this.stopped) return;
|
|
2074
|
-
const asyncId = this.promiseIds.get(promise);
|
|
2075
|
-
if (asyncId == null || this.resources.get(asyncId)?.pending !== true || this.nativeDurableAssimilations.has(asyncId) || this.explicitDurableAssimilations.has(asyncId)) {
|
|
2076
|
-
return;
|
|
2077
|
-
}
|
|
2078
|
-
this.explicitDurableAssimilations.add(asyncId);
|
|
2079
|
-
this.promiseAggregateAssimilationScopes.at(-1)?.add(asyncId);
|
|
2080
|
-
this.suspendDurableAssimilation(asyncId);
|
|
2081
|
-
}
|
|
2082
|
-
runPromiseAggregateConstruction(callback) {
|
|
2083
|
-
const aggregateAssimilations = /* @__PURE__ */ new Set();
|
|
2084
|
-
this.promiseAggregateAssimilationScopes.push(aggregateAssimilations);
|
|
2085
|
-
try {
|
|
2086
|
-
return callback();
|
|
2087
|
-
} finally {
|
|
2088
|
-
this.promiseAggregateAssimilationScopes.pop();
|
|
2089
|
-
this.detachPromiseAggregateAssimilations(aggregateAssimilations);
|
|
2090
|
-
}
|
|
2091
|
-
}
|
|
2092
|
-
runWithoutOwnership(callback) {
|
|
2093
|
-
if (this.stopped) return callback();
|
|
2094
|
-
const previous = this.suppressOwnership;
|
|
2095
|
-
this.suppressOwnership = true;
|
|
2096
|
-
try {
|
|
2097
|
-
return callback();
|
|
2098
|
-
} finally {
|
|
2099
|
-
this.suppressOwnership = previous;
|
|
2100
|
-
}
|
|
2101
|
-
}
|
|
2102
|
-
stop() {
|
|
2103
|
-
if (this.stopped) return;
|
|
2104
|
-
this.stopped = true;
|
|
2105
|
-
this.hook.disable();
|
|
2106
|
-
if (this.checkHandle != null) clearImmediate(this.checkHandle);
|
|
2107
|
-
this.checkHandle = void 0;
|
|
2108
|
-
this.children.clear();
|
|
2109
|
-
this.resources.clear();
|
|
2110
|
-
this.owners.clear();
|
|
2111
|
-
this.provenance.clear();
|
|
2112
|
-
this.tokenResources.clear();
|
|
2113
|
-
this.tokenProducers.clear();
|
|
2114
|
-
this.durableAssimilations.clear();
|
|
2115
|
-
this.nativeDurableAssimilations.clear();
|
|
2116
|
-
this.explicitDurableAssimilations.clear();
|
|
2117
|
-
this.promiseAggregateAssimilationScopes.length = 0;
|
|
2118
|
-
this.promiseSettlementObservers.clear();
|
|
2119
|
-
this.mixedPromiseProducerGroups.clear();
|
|
2120
|
-
}
|
|
2121
|
-
notifyPromiseSettlement(asyncId) {
|
|
2122
|
-
const observers = this.promiseSettlementObservers.get(asyncId);
|
|
2123
|
-
if (observers == null) return;
|
|
2124
|
-
this.promiseSettlementObservers.delete(asyncId);
|
|
2125
|
-
for (const observer of observers) observer();
|
|
2126
|
-
}
|
|
2127
|
-
settleMixedPromiseProducerGroups(asyncId) {
|
|
2128
|
-
for (const group of [...this.mixedPromiseProducerGroups]) {
|
|
2129
|
-
if (group.has(asyncId)) this.mixedPromiseProducerGroups.delete(group);
|
|
2130
|
-
}
|
|
2131
|
-
}
|
|
2132
|
-
detachPromiseAggregateAssimilations(assimilations) {
|
|
2133
|
-
for (const asyncId of assimilations) {
|
|
2134
|
-
if (this.nativeDurableAssimilations.delete(asyncId)) {
|
|
2135
|
-
this.detachDurableAssimilation(asyncId);
|
|
2136
|
-
}
|
|
2137
|
-
if (this.explicitDurableAssimilations.delete(asyncId)) {
|
|
2138
|
-
this.detachDurableAssimilation(asyncId);
|
|
2139
|
-
}
|
|
2140
|
-
}
|
|
2141
|
-
assimilations.clear();
|
|
2142
|
-
}
|
|
2143
|
-
addTokenToSubtree(asyncId, token) {
|
|
2144
|
-
if (this.excludedRoots.has(asyncId)) return;
|
|
2145
|
-
const owners = this.owners.get(asyncId) ?? /* @__PURE__ */ new Set();
|
|
2146
|
-
if (!owners.has(token)) {
|
|
2147
|
-
owners.add(token);
|
|
2148
|
-
this.owners.set(asyncId, owners);
|
|
2149
|
-
const provenance = this.provenance.get(asyncId) ?? /* @__PURE__ */ new Set();
|
|
2150
|
-
provenance.add(token);
|
|
2151
|
-
this.provenance.set(asyncId, provenance);
|
|
2152
|
-
const tokenResources = this.tokenResources.get(token) ?? /* @__PURE__ */ new Set();
|
|
2153
|
-
tokenResources.add(asyncId);
|
|
2154
|
-
this.tokenResources.set(token, tokenResources);
|
|
2155
|
-
const resource = this.resources.get(asyncId);
|
|
2156
|
-
if (resource?.pending && this.isProducer(resource, token)) {
|
|
2157
|
-
const producers = this.tokenProducers.get(token) ?? /* @__PURE__ */ new Set();
|
|
2158
|
-
producers.add(asyncId);
|
|
2159
|
-
this.tokenProducers.set(token, producers);
|
|
2160
|
-
}
|
|
2161
|
-
}
|
|
2162
|
-
for (const child of this.children.get(asyncId) ?? []) {
|
|
2163
|
-
this.addTokenToSubtree(child, token);
|
|
2164
|
-
}
|
|
2165
|
-
}
|
|
2166
|
-
isProducer(resource, token) {
|
|
2167
|
-
if (resource.type === "PROMISE") {
|
|
2168
|
-
return !(this.provenance.get(resource.triggerAsyncId)?.has(token) ?? false);
|
|
2169
|
-
}
|
|
2170
|
-
return !PASSIVE_ASYNC_RESOURCE_TYPES.has(resource.type);
|
|
2171
|
-
}
|
|
2172
|
-
setResourcePending(asyncId, pending) {
|
|
2173
|
-
const resource = this.resources.get(asyncId);
|
|
2174
|
-
if (resource == null || resource.pending === pending) return;
|
|
2175
|
-
resource.pending = pending;
|
|
2176
|
-
for (const token of this.owners.get(asyncId) ?? []) {
|
|
2177
|
-
const producers = this.tokenProducers.get(token);
|
|
2178
|
-
if (pending && this.isProducer(resource, token)) {
|
|
2179
|
-
const next = producers ?? /* @__PURE__ */ new Set();
|
|
2180
|
-
next.add(asyncId);
|
|
2181
|
-
this.tokenProducers.set(token, next);
|
|
2182
|
-
} else {
|
|
2183
|
-
producers?.delete(asyncId);
|
|
2184
|
-
}
|
|
2185
|
-
}
|
|
2186
|
-
}
|
|
2187
|
-
removeOwnersFromSubtree(asyncId) {
|
|
2188
|
-
this.removeResourceOwnership(asyncId);
|
|
2189
|
-
for (const child of this.children.get(asyncId) ?? []) {
|
|
2190
|
-
this.removeOwnersFromSubtree(child);
|
|
2191
|
-
}
|
|
2192
|
-
}
|
|
2193
|
-
removeResourceOwnership(asyncId) {
|
|
2194
|
-
const owners = this.owners.get(asyncId);
|
|
2195
|
-
if (owners == null) return;
|
|
2196
|
-
for (const token of owners) {
|
|
2197
|
-
const resources = this.tokenResources.get(token);
|
|
2198
|
-
resources?.delete(asyncId);
|
|
2199
|
-
if (resources?.size === 0) this.tokenResources.delete(token);
|
|
2200
|
-
const producers = this.tokenProducers.get(token);
|
|
2201
|
-
producers?.delete(asyncId);
|
|
2202
|
-
if (producers?.size === 0) this.tokenProducers.delete(token);
|
|
2203
|
-
}
|
|
2204
|
-
this.owners.delete(asyncId);
|
|
2205
|
-
}
|
|
2206
|
-
releaseDestroyedResource(asyncId) {
|
|
2207
|
-
const resource = this.resources.get(asyncId);
|
|
2208
|
-
if (resource == null) return;
|
|
2209
|
-
this.resources.delete(asyncId);
|
|
2210
|
-
this.excludedRoots.delete(asyncId);
|
|
2211
|
-
this.removeResourceOwnership(asyncId);
|
|
2212
|
-
this.provenance.delete(asyncId);
|
|
2213
|
-
const siblings = this.children.get(resource.triggerAsyncId);
|
|
2214
|
-
siblings?.delete(asyncId);
|
|
2215
|
-
if (siblings?.size === 0) this.children.delete(resource.triggerAsyncId);
|
|
2216
|
-
this.children.delete(asyncId);
|
|
2217
|
-
}
|
|
2218
|
-
scheduleCheck() {
|
|
2219
|
-
if (this.stopped || this.checkScheduled) return;
|
|
2220
|
-
this.checkScheduled = true;
|
|
2221
|
-
const previous = this.suppressOwnership;
|
|
2222
|
-
this.suppressOwnership = true;
|
|
2223
|
-
try {
|
|
2224
|
-
this.checkHandle = setImmediate(() => {
|
|
2225
|
-
this.checkScheduled = false;
|
|
2226
|
-
this.checkHandle = void 0;
|
|
2227
|
-
this.onQuiescent();
|
|
2228
|
-
});
|
|
2229
|
-
} finally {
|
|
2230
|
-
this.suppressOwnership = previous;
|
|
2231
|
-
}
|
|
2232
|
-
}
|
|
2233
|
-
};
|
|
2234
|
-
var ReplayAwarePromise = class extends Promise {
|
|
2235
|
-
causality;
|
|
2236
|
-
constructor(source, causality) {
|
|
2237
|
-
let bridge;
|
|
2238
|
-
super((resolve, reject) => {
|
|
2239
|
-
bridge = source.then(resolve, reject);
|
|
2240
|
-
});
|
|
2241
|
-
this.causality = causality;
|
|
2242
|
-
registerDurableThenable(this);
|
|
2243
|
-
causality.excludePromise(source);
|
|
2244
|
-
causality.excludePromise(this);
|
|
2245
|
-
if (bridge != null) causality.excludePromise(bridge);
|
|
2246
|
-
}
|
|
2247
|
-
static get [Symbol.species]() {
|
|
2248
|
-
return Promise;
|
|
2249
|
-
}
|
|
2250
|
-
then(onfulfilled, onrejected) {
|
|
2251
|
-
const assimilationId = (0, import_node_async_hooks3.executionAsyncId)();
|
|
2252
|
-
const assimilationSuspended = this.causality.suspendDurableAssimilation(assimilationId);
|
|
2253
|
-
let assimilationDetached = false;
|
|
2254
|
-
const aggregateAssimilation = registerDurableAssimilation(this, () => {
|
|
2255
|
-
if (!assimilationSuspended || assimilationDetached) return;
|
|
2256
|
-
assimilationDetached = true;
|
|
2257
|
-
this.causality.detachDurableAssimilation(assimilationId);
|
|
2258
|
-
});
|
|
2259
|
-
const continuation = super.then(
|
|
2260
|
-
(value) => {
|
|
2261
|
-
this.causality.beginCurrentContinuation();
|
|
2262
|
-
aggregateAssimilation.finish();
|
|
2263
|
-
if (assimilationSuspended && !assimilationDetached) {
|
|
2264
|
-
this.causality.resumeDurableAssimilation(assimilationId);
|
|
2265
|
-
}
|
|
2266
|
-
if (onfulfilled == null) return value;
|
|
2267
|
-
return onfulfilled(value);
|
|
2268
|
-
},
|
|
2269
|
-
(reason) => {
|
|
2270
|
-
this.causality.beginCurrentContinuation();
|
|
2271
|
-
aggregateAssimilation.finish();
|
|
2272
|
-
if (assimilationSuspended && !assimilationDetached) {
|
|
2273
|
-
this.causality.resumeDurableAssimilation(assimilationId);
|
|
2274
|
-
}
|
|
2275
|
-
if (onrejected == null) throw reason;
|
|
2276
|
-
return onrejected(reason);
|
|
2277
|
-
}
|
|
2278
|
-
);
|
|
2279
|
-
aggregateAssimilation.link(continuation);
|
|
2280
|
-
this.causality.excludePromise(continuation);
|
|
2281
|
-
return continuation;
|
|
2282
|
-
}
|
|
2283
|
-
[DURABLE_PROMISE_INTERNAL_THEN](onfulfilled, onrejected) {
|
|
2284
|
-
const continuation = super.then(onfulfilled, onrejected);
|
|
2285
|
-
registerDurableThenable(continuation, this);
|
|
2286
|
-
this.causality.excludePromise(continuation);
|
|
2287
|
-
return continuation;
|
|
2288
|
-
}
|
|
2289
|
-
[DURABLE_PROMISE_OBSERVER](onfulfilled, onrejected) {
|
|
2290
|
-
const observation = super.then(onfulfilled, onrejected);
|
|
2291
|
-
this.causality.excludePromise(observation);
|
|
2292
|
-
void observation.catch(() => void 0);
|
|
2293
|
-
}
|
|
2294
|
-
};
|
|
2295
|
-
var ReplayHistory = class {
|
|
2296
|
-
entries;
|
|
2297
|
-
calls = [];
|
|
2298
|
-
watchers = [];
|
|
2299
|
-
availableResults = /* @__PURE__ */ new Map();
|
|
2300
|
-
resultWaiters = /* @__PURE__ */ new Map();
|
|
2301
|
-
endWaiters = [];
|
|
2302
|
-
createdWatcherIds = /* @__PURE__ */ new Set();
|
|
2303
|
-
releasedResultIds = /* @__PURE__ */ new Set();
|
|
2304
|
-
expectedCallIds = /* @__PURE__ */ new Set();
|
|
2305
|
-
expectedWatcherIds = /* @__PURE__ */ new Set();
|
|
2306
|
-
causality;
|
|
2307
|
-
// A replayed result may still be downloading before it can resume user code.
|
|
2308
|
-
activeResultResolutions = 0;
|
|
2309
|
-
callCursor = 0;
|
|
2310
|
-
watcherCursor = 0;
|
|
2311
|
-
prefixCursor = 0;
|
|
2312
|
-
mismatch;
|
|
2313
|
-
constructor(entries) {
|
|
2314
|
-
this.causality = new ReplayCausality(() => this.failBlockedResultWithoutProducer());
|
|
2315
|
-
this.entries = entries.map((entry) => {
|
|
2316
|
-
if (entry.functionCallWatcherResult != null) {
|
|
2317
|
-
return { kind: "result", event: entry.functionCallWatcherResult, consumed: false };
|
|
2318
|
-
}
|
|
2319
|
-
if (entry.functionCallCreated != null) {
|
|
2320
|
-
const replayEntry = {
|
|
2321
|
-
kind: "call",
|
|
2322
|
-
event: entry.functionCallCreated,
|
|
2323
|
-
consumed: false
|
|
2324
|
-
};
|
|
2325
|
-
this.calls.push(replayEntry);
|
|
2326
|
-
return replayEntry;
|
|
2327
|
-
}
|
|
2328
|
-
if (entry.functionCallWatcherCreated != null) {
|
|
2329
|
-
const replayEntry = {
|
|
2330
|
-
kind: "watcher",
|
|
2331
|
-
event: entry.functionCallWatcherCreated,
|
|
2332
|
-
consumed: false
|
|
2333
|
-
};
|
|
2334
|
-
this.watchers.push(replayEntry);
|
|
2335
|
-
return replayEntry;
|
|
2336
|
-
}
|
|
2337
|
-
return { kind: "unknown", event: entry, consumed: false };
|
|
2338
|
-
});
|
|
2339
|
-
}
|
|
2340
|
-
async takeOrdered(kind, id) {
|
|
2341
|
-
this.advancePrefix();
|
|
2342
|
-
const entries = kind === "call" ? this.calls : this.watchers;
|
|
2343
|
-
const cursor = kind === "call" ? this.callCursor : this.watcherCursor;
|
|
2344
|
-
const expectedIds = kind === "call" ? this.expectedCallIds : this.expectedWatcherIds;
|
|
2345
|
-
if (cursor >= entries.length) {
|
|
2346
|
-
expectedIds.delete(id);
|
|
2347
|
-
if (this.prefixCursor === this.entries.length) return void 0;
|
|
2348
|
-
const blocked = this.entries[this.prefixCursor];
|
|
2349
|
-
if (kind !== "call" || blocked.kind !== "watcher" || !this.expectedWatcherIds.has(String(blocked.event.functionCallId ?? ""))) {
|
|
2350
|
-
this.fail(
|
|
2351
|
-
`Unexpected ${kind} event for ${id} before replay ${blocked.kind} event ${String(blocked.event.functionCallId ?? "")}`
|
|
2352
|
-
);
|
|
2353
|
-
}
|
|
2354
|
-
const waiter = deferred();
|
|
2355
|
-
this.endWaiters.push(waiter);
|
|
2356
|
-
await waiter.promise;
|
|
2357
|
-
this.assertNoMismatch();
|
|
2358
|
-
return void 0;
|
|
2359
|
-
}
|
|
2360
|
-
const entry = entries[cursor];
|
|
2361
|
-
if (kind === "call") this.callCursor += 1;
|
|
2362
|
-
else this.watcherCursor += 1;
|
|
2363
|
-
if (entry.event.functionCallId !== id) {
|
|
2364
|
-
this.fail(`Expected replay ${kind} event for ${id}`);
|
|
2365
|
-
}
|
|
2366
|
-
expectedIds.delete(id);
|
|
2367
|
-
entry.consumed = true;
|
|
2368
|
-
if (kind === "watcher") {
|
|
2369
|
-
this.createdWatcherIds.add(id);
|
|
2370
|
-
}
|
|
2371
|
-
this.advancePrefix();
|
|
2372
|
-
return entry.event;
|
|
2373
|
-
}
|
|
2374
|
-
async takeResult(id) {
|
|
2375
|
-
this.advancePrefix();
|
|
2376
|
-
const available = this.availableResults.get(id);
|
|
2377
|
-
if (available != null && available.length > 0) {
|
|
2378
|
-
const result2 = available.shift();
|
|
2379
|
-
if (result2 != null) this.activeResultResolutions += 1;
|
|
2380
|
-
return result2;
|
|
2381
|
-
}
|
|
2382
|
-
if (this.prefixCursor === this.entries.length) return void 0;
|
|
2383
|
-
const blocked = this.entries[this.prefixCursor];
|
|
2384
|
-
const blockedId = String(blocked.event.functionCallId ?? "");
|
|
2385
|
-
const hasProducer = blocked.kind === "call" ? this.expectedCallIds.has(blockedId) : blocked.kind === "watcher" ? this.expectedWatcherIds.has(blockedId) : false;
|
|
2386
|
-
const waiter = deferred();
|
|
2387
|
-
const waiters = this.resultWaiters.get(id) ?? [];
|
|
2388
|
-
waiters.push(waiter);
|
|
2389
|
-
this.resultWaiters.set(id, waiters);
|
|
2390
|
-
if (!hasProducer) this.causality.requestCheck();
|
|
2391
|
-
const result = await waiter.promise;
|
|
2392
|
-
if (result != null) this.activeResultResolutions += 1;
|
|
2393
|
-
return result;
|
|
2394
|
-
}
|
|
2395
|
-
completeResultResolution() {
|
|
2396
|
-
if (this.activeResultResolutions === 0) return;
|
|
2397
|
-
this.activeResultResolutions -= 1;
|
|
2398
|
-
this.causality.requestCheck();
|
|
2399
|
-
}
|
|
2400
|
-
runCausalHandler(callback) {
|
|
2401
|
-
return this.causality.runRoot(callback);
|
|
2402
|
-
}
|
|
2403
|
-
wrapDurableResult(source) {
|
|
2404
|
-
return new ReplayAwarePromise(source, this.causality);
|
|
2405
|
-
}
|
|
2406
|
-
trackMixedPromiseRace(promise, producers) {
|
|
2407
|
-
this.causality.trackMixedPromiseRace(promise, producers);
|
|
2408
|
-
}
|
|
2409
|
-
trackPromiseSettlement(promise, onSettled) {
|
|
2410
|
-
return this.causality.trackPromiseSettlement(promise, onSettled);
|
|
2411
|
-
}
|
|
2412
|
-
trackDurablePromiseConsumption(promise) {
|
|
2413
|
-
this.causality.trackDurablePromiseConsumption(promise);
|
|
2414
|
-
}
|
|
2415
|
-
runPromiseAggregateConstruction(callback) {
|
|
2416
|
-
return this.causality.runPromiseAggregateConstruction(callback);
|
|
2417
|
-
}
|
|
2418
|
-
runWithoutCausalOwnership(callback) {
|
|
2419
|
-
return this.causality.runWithoutOwnership(callback);
|
|
2420
|
-
}
|
|
2421
|
-
dispose() {
|
|
2422
|
-
this.causality.stop();
|
|
2423
|
-
}
|
|
2424
|
-
expectCall(id) {
|
|
2425
|
-
this.expectedCallIds.add(id);
|
|
2426
|
-
}
|
|
2427
|
-
expectWatcher(id) {
|
|
2428
|
-
this.expectedWatcherIds.add(id);
|
|
2429
|
-
}
|
|
2430
|
-
abandonCall(id) {
|
|
2431
|
-
this.abandonOrdered("call", id);
|
|
2432
|
-
}
|
|
2433
|
-
abandonWatcher(id) {
|
|
2434
|
-
this.abandonOrdered("watcher", id);
|
|
2435
|
-
}
|
|
2436
|
-
abandonOrdered(kind, id) {
|
|
2437
|
-
const expectedIds = kind === "call" ? this.expectedCallIds : this.expectedWatcherIds;
|
|
2438
|
-
expectedIds.delete(id);
|
|
2439
|
-
const hasWaiters = this.endWaiters.length > 0 || [...this.resultWaiters.values()].some((waiters) => waiters.length > 0);
|
|
2440
|
-
if (this.prefixCursor >= this.entries.length || !hasWaiters) return;
|
|
2441
|
-
const blocked = this.entries[this.prefixCursor];
|
|
2442
|
-
if (blocked.kind === kind && blocked.event.functionCallId === id) {
|
|
2443
|
-
this.fail(`Replay ${kind} event for ${id} can no longer be created`);
|
|
2444
|
-
}
|
|
2445
|
-
}
|
|
2446
|
-
failBlockedResultWithoutProducer() {
|
|
2447
|
-
if (this.activeResultResolutions > 0) return;
|
|
2448
|
-
if (!this.causality.hasPendingDurableAssimilation()) return;
|
|
2449
|
-
if (this.causality.hasPendingProducer()) return;
|
|
2450
|
-
if (this.prefixCursor >= this.entries.length) return;
|
|
2451
|
-
if (![...this.resultWaiters.values()].some((waiters) => waiters.length > 0)) return;
|
|
2452
|
-
const blocked = this.entries[this.prefixCursor];
|
|
2453
|
-
const blockedId = String(blocked.event.functionCallId ?? "");
|
|
2454
|
-
const hasProducer = blocked.kind === "call" ? this.expectedCallIds.has(blockedId) : blocked.kind === "watcher" ? this.expectedWatcherIds.has(blockedId) : false;
|
|
2455
|
-
if (!hasProducer) {
|
|
2456
|
-
this.recordMismatch(
|
|
2457
|
-
`Replay result is blocked behind unreachable ${blocked.kind} event ${blockedId}`
|
|
2458
|
-
);
|
|
2459
|
-
}
|
|
2460
|
-
}
|
|
2461
|
-
hasAvailableResult(id) {
|
|
2462
|
-
return (this.availableResults.get(id)?.length ?? 0) > 0;
|
|
2463
|
-
}
|
|
2464
|
-
assertNoMismatch() {
|
|
2465
|
-
if (this.mismatch != null) throw this.mismatch;
|
|
2466
|
-
}
|
|
2467
|
-
assertConsumed() {
|
|
2468
|
-
this.assertNoMismatch();
|
|
2469
|
-
this.advancePrefix();
|
|
2470
|
-
if (this.prefixCursor !== this.entries.length) {
|
|
2471
|
-
throw new ReplayMismatchError("Function completed before its durable event history was consumed");
|
|
2472
|
-
}
|
|
2473
|
-
}
|
|
2474
|
-
advancePrefix() {
|
|
2475
|
-
this.assertNoMismatch();
|
|
2476
|
-
while (this.prefixCursor < this.entries.length) {
|
|
2477
|
-
const entry = this.entries[this.prefixCursor];
|
|
2478
|
-
if (entry.kind === "unknown") {
|
|
2479
|
-
this.fail("Replay history contains an unknown allocation event");
|
|
2480
|
-
}
|
|
2481
|
-
if (entry.kind === "call" || entry.kind === "watcher") {
|
|
2482
|
-
if (!entry.consumed) return;
|
|
2483
|
-
this.prefixCursor += 1;
|
|
2484
|
-
continue;
|
|
2485
|
-
}
|
|
2486
|
-
entry.consumed = true;
|
|
2487
|
-
this.prefixCursor += 1;
|
|
2488
|
-
this.releaseResult(entry.event);
|
|
2489
|
-
}
|
|
2490
|
-
for (const waiter of this.endWaiters.splice(0)) waiter.resolve();
|
|
2491
|
-
for (const waiters of this.resultWaiters.values()) {
|
|
2492
|
-
for (const waiter of waiters) waiter.resolve(void 0);
|
|
2493
|
-
}
|
|
2494
|
-
this.resultWaiters.clear();
|
|
2495
|
-
}
|
|
2496
|
-
releaseResult(event) {
|
|
2497
|
-
const id = String(event.functionCallId ?? "");
|
|
2498
|
-
if (!this.createdWatcherIds.has(id)) {
|
|
2499
|
-
this.fail(`Replay watcher result for ${id} appeared before its watcher was created`);
|
|
2500
|
-
}
|
|
2501
|
-
if (this.releasedResultIds.has(id)) {
|
|
2502
|
-
this.fail(`Replay history contains duplicate watcher results for ${id}`);
|
|
2503
|
-
}
|
|
2504
|
-
this.releasedResultIds.add(id);
|
|
2505
|
-
const waiter = this.resultWaiters.get(id)?.shift();
|
|
2506
|
-
if (waiter != null) {
|
|
2507
|
-
waiter.resolve(event);
|
|
2508
|
-
if (this.resultWaiters.get(id)?.length === 0) this.resultWaiters.delete(id);
|
|
2509
|
-
return;
|
|
2510
|
-
}
|
|
2511
|
-
const available = this.availableResults.get(id) ?? [];
|
|
2512
|
-
available.push(event);
|
|
2513
|
-
this.availableResults.set(id, available);
|
|
2514
|
-
}
|
|
2515
|
-
fail(message) {
|
|
2516
|
-
throw this.recordMismatch(message);
|
|
2517
|
-
}
|
|
2518
|
-
recordMismatch(message) {
|
|
2519
|
-
this.mismatch ??= new ReplayMismatchError(message);
|
|
2520
|
-
this.causality.stop();
|
|
2521
|
-
for (const waiter of this.endWaiters.splice(0)) waiter.reject(this.mismatch);
|
|
2522
|
-
for (const waiters of this.resultWaiters.values()) {
|
|
2523
|
-
for (const waiter of waiters) waiter.reject(this.mismatch);
|
|
2524
|
-
}
|
|
2525
|
-
this.resultWaiters.clear();
|
|
2526
|
-
return this.mismatch;
|
|
2527
|
-
}
|
|
2528
|
-
};
|
|
2529
|
-
var AllocationProtocolError = class extends Error {
|
|
2530
|
-
constructor(message, options) {
|
|
2531
|
-
super(message, options);
|
|
2532
|
-
this.name = "AllocationProtocolError";
|
|
2533
|
-
}
|
|
2534
|
-
};
|
|
2535
|
-
var AllocationRunner = class {
|
|
2536
|
-
allocation;
|
|
2537
|
-
functionRef;
|
|
2538
|
-
definition;
|
|
2539
|
-
applicationDefinition;
|
|
2540
|
-
stateStreams = /* @__PURE__ */ new Set();
|
|
2541
|
-
stateStreamFlows = /* @__PURE__ */ new Map();
|
|
2542
|
-
eventReadStreams = /* @__PURE__ */ new Set();
|
|
2543
|
-
outputBlobRequests = /* @__PURE__ */ new Map();
|
|
2544
|
-
stateOperationRequests = /* @__PURE__ */ new Map();
|
|
2545
|
-
executionBatches = [];
|
|
2546
|
-
executionBatchWaiters = [];
|
|
2547
|
-
liveBacklog = [];
|
|
2548
|
-
liveWaiters = [];
|
|
2549
|
-
controller = new AbortController();
|
|
2550
|
-
currentRead;
|
|
2551
|
-
replay;
|
|
2552
|
-
protocolError;
|
|
2553
|
-
durableCallEmissionTail = Promise.resolve();
|
|
2554
|
-
durableWatcherEmissionTail = Promise.resolve();
|
|
2555
|
-
lastEventClock = 0;
|
|
2556
|
-
previousDurableId;
|
|
2557
|
-
livePump;
|
|
2558
|
-
completion;
|
|
2559
|
-
started = false;
|
|
2560
|
-
terminalBatchQueued = false;
|
|
2561
|
-
terminalBatchObserved = false;
|
|
2562
|
-
terminalBatchObservedWaiters = [];
|
|
2563
|
-
finished = false;
|
|
2564
|
-
state = { outputBlobRequests: [], requestStateOperations: [] };
|
|
2565
|
-
startedAt = Date.now();
|
|
2566
|
-
constructor(allocation, functionRef, definition, applicationDefinition = definition) {
|
|
2567
|
-
this.allocation = allocation;
|
|
2568
|
-
this.functionRef = functionRef;
|
|
2569
|
-
this.definition = definition;
|
|
2570
|
-
this.applicationDefinition = applicationDefinition;
|
|
2571
|
-
this.previousDurableId = String(allocation.functionCallId);
|
|
2572
|
-
this.updateStateHash();
|
|
2573
|
-
this.log("debug", "allocation runner created", {
|
|
2574
|
-
replay_mode: allocation.replayMode,
|
|
2575
|
-
input_count: allocation.inputs?.args?.length ?? 0,
|
|
2576
|
-
input_blob_count: allocation.inputs?.argBlobs?.length ?? 0
|
|
2577
|
-
});
|
|
2578
|
-
}
|
|
2579
|
-
start() {
|
|
2580
|
-
if (this.started) {
|
|
2581
|
-
this.log("warn", "duplicate allocation runner start ignored");
|
|
2582
|
-
return;
|
|
2583
|
-
}
|
|
2584
|
-
this.started = true;
|
|
2585
|
-
this.log("info", "allocation runner scheduled");
|
|
2586
|
-
this.completion = this.run().catch((error) => {
|
|
2587
|
-
this.log("error", "allocation runner promise rejected", {}, error);
|
|
2588
|
-
this.queueTerminalBatch({
|
|
2589
|
-
outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
|
|
2590
|
-
failureReason: "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR"
|
|
2591
|
-
});
|
|
2592
|
-
this.finish();
|
|
2593
|
-
});
|
|
2594
|
-
}
|
|
2595
|
-
waitForCompletion() {
|
|
2596
|
-
return this.completion ?? Promise.resolve();
|
|
2597
|
-
}
|
|
2598
|
-
async waitForTerminalBatchObserved(timeoutMs) {
|
|
2599
|
-
if (this.terminalBatchObserved) return true;
|
|
2600
|
-
return new Promise((resolve) => {
|
|
2601
|
-
let settled = false;
|
|
2602
|
-
const observed = () => {
|
|
2603
|
-
if (settled) return;
|
|
2604
|
-
settled = true;
|
|
2605
|
-
clearTimeout(timer);
|
|
2606
|
-
resolve(true);
|
|
2607
|
-
};
|
|
2608
|
-
const timer = setTimeout(() => {
|
|
2609
|
-
if (settled) return;
|
|
2610
|
-
settled = true;
|
|
2611
|
-
const index = this.terminalBatchObservedWaiters.indexOf(observed);
|
|
2612
|
-
if (index >= 0) this.terminalBatchObservedWaiters.splice(index, 1);
|
|
2613
|
-
resolve(false);
|
|
2614
|
-
}, Math.max(timeoutMs, 0));
|
|
2615
|
-
this.terminalBatchObservedWaiters.push(observed);
|
|
2616
|
-
if (this.terminalBatchObserved) observed();
|
|
2617
|
-
});
|
|
2618
|
-
}
|
|
2619
|
-
cancel(reason = new FunctionError("Allocation was cancelled")) {
|
|
2620
|
-
if (this.controller.signal.aborted) return;
|
|
2621
|
-
this.log("info", "allocation cancellation requested", {
|
|
2622
|
-
reason: reason instanceof Error ? reason.message : String(reason)
|
|
2623
|
-
});
|
|
2624
|
-
this.abortExecution(reason);
|
|
2625
|
-
}
|
|
2626
|
-
abortExecution(reason) {
|
|
2627
|
-
if (!this.controller.signal.aborted) this.controller.abort(reason);
|
|
2628
|
-
const rejection = this.controller.signal.reason ?? reason;
|
|
2629
|
-
const outputBlobRequestCount = this.outputBlobRequests.size;
|
|
2630
|
-
const stateOperationRequestCount = this.stateOperationRequests.size;
|
|
2631
|
-
const liveWaiterCount = this.liveWaiters.length;
|
|
2632
|
-
const liveBacklogCount = this.liveBacklog.length;
|
|
2633
|
-
const hadCurrentRead = this.currentRead != null;
|
|
2634
|
-
for (const request of this.outputBlobRequests.values()) request.reject(rejection);
|
|
2635
|
-
for (const request of this.stateOperationRequests.values()) request.reject(rejection);
|
|
2636
|
-
for (const waiter of this.liveWaiters.splice(0)) waiter.result.reject(rejection);
|
|
2637
|
-
this.currentRead?.response.reject(rejection);
|
|
2638
|
-
this.outputBlobRequests.clear();
|
|
2639
|
-
this.stateOperationRequests.clear();
|
|
2640
|
-
this.liveBacklog.splice(0);
|
|
2641
|
-
this.currentRead = void 0;
|
|
2642
|
-
this.state.outputBlobRequests = [];
|
|
2643
|
-
this.state.requestStateOperations = [];
|
|
2644
|
-
this.log("debug", "pending allocation protocol work settled", {
|
|
2645
|
-
output_blob_requests: outputBlobRequestCount,
|
|
2646
|
-
state_operation_requests: stateOperationRequestCount,
|
|
2647
|
-
durable_event_waiters: liveWaiterCount,
|
|
2648
|
-
durable_event_backlog_entries: liveBacklogCount,
|
|
2649
|
-
event_log_read: hadCurrentRead
|
|
2650
|
-
});
|
|
2651
|
-
}
|
|
2652
|
-
invoke(definition, args) {
|
|
2653
|
-
this.log("debug", "durable function invocation requested", {
|
|
2654
|
-
target_function: definition.name,
|
|
2655
|
-
argument_count: args.length
|
|
2656
|
-
});
|
|
2657
|
-
const result = this.runChild(definition, args, 0);
|
|
2658
|
-
return this.replay?.wrapDurableResult(result) ?? result;
|
|
2659
|
-
}
|
|
2660
|
-
runFuture(future) {
|
|
2661
|
-
this.log("debug", "durable future execution requested", {
|
|
2662
|
-
target_function: future.definition.name,
|
|
2663
|
-
argument_count: future.args.length,
|
|
2664
|
-
delay_seconds: future.delaySeconds
|
|
2665
|
-
});
|
|
2666
|
-
const result = this.runChild(future.definition, future.args, future.delaySeconds);
|
|
2667
|
-
return this.replay?.wrapDurableResult(result) ?? result;
|
|
2668
|
-
}
|
|
2669
|
-
wrapFutureWait(promise) {
|
|
2670
|
-
return this.replay?.wrapDurableResult(promise) ?? promise;
|
|
2671
|
-
}
|
|
2672
|
-
trackMixedPromiseRace(promise, producers) {
|
|
2673
|
-
this.replay?.trackMixedPromiseRace(promise, producers);
|
|
2674
|
-
}
|
|
2675
|
-
trackPromiseSettlement(promise, onSettled) {
|
|
2676
|
-
return this.replay?.trackPromiseSettlement(promise, onSettled) ?? false;
|
|
2677
|
-
}
|
|
2678
|
-
trackDurablePromiseConsumption(promise) {
|
|
2679
|
-
this.replay?.trackDurablePromiseConsumption(promise);
|
|
2680
|
-
}
|
|
2681
|
-
runPromiseAggregateConstruction(callback) {
|
|
2682
|
-
return this.replay?.runPromiseAggregateConstruction(callback) ?? callback();
|
|
2683
|
-
}
|
|
2684
|
-
reduce(definition, items, initial, hasInitial) {
|
|
2685
|
-
const result = this.runReduce(definition, items, initial, hasInitial);
|
|
2686
|
-
return this.replay?.wrapDurableResult(result) ?? result;
|
|
2687
|
-
}
|
|
2688
|
-
async runReduce(definition, items, initial, hasInitial) {
|
|
2689
|
-
let reduceItems = items;
|
|
2690
|
-
let reduceInitial = initial;
|
|
2691
|
-
if (!hasInitial) {
|
|
2692
|
-
if (items.length === 0) {
|
|
2693
|
-
throw new SDKUsageError("reduce of empty iterable with no initial value");
|
|
2694
|
-
}
|
|
2695
|
-
[reduceInitial, ...reduceItems] = items;
|
|
2696
|
-
}
|
|
2697
|
-
if (reduceItems.length === 0) return reduceInitial;
|
|
2698
|
-
const callTurn = this.reserveEmissionTurn("call");
|
|
2699
|
-
const watcherTurn = this.reserveEmissionTurn("watcher");
|
|
2700
|
-
const ids = reduceItems.map(() => this.nextDurableId(definition, "ReduceStep"));
|
|
2701
|
-
const id = ids[ids.length - 1];
|
|
2702
|
-
const startedAt = Date.now();
|
|
2703
|
-
this.log("info", "durable reduce starting", {
|
|
2704
|
-
durable_id: id,
|
|
2705
|
-
target_function: definition.name,
|
|
2706
|
-
collection_size: reduceItems.length,
|
|
2707
|
-
plan_count: Math.ceil(reduceItems.length / MAX_REDUCE_CALLS_PER_PLAN)
|
|
2708
|
-
});
|
|
2709
|
-
const replayCallIds = ids.filter(
|
|
2710
|
-
(_id, index) => (index + 1) % MAX_REDUCE_CALLS_PER_PLAN === 0 || index === ids.length - 1
|
|
2711
|
-
);
|
|
2712
|
-
for (const replayCallId of replayCallIds) this.replay?.expectCall(replayCallId);
|
|
2713
|
-
this.replay?.expectWatcher(id);
|
|
2714
|
-
try {
|
|
2715
|
-
try {
|
|
2716
|
-
await this.createReduceChain(ids, definition, reduceItems, reduceInitial, callTurn);
|
|
2717
|
-
} catch (error) {
|
|
2718
|
-
for (const replayCallId of replayCallIds) this.replay?.abandonCall(replayCallId);
|
|
2719
|
-
this.replay?.abandonWatcher(id);
|
|
2720
|
-
throw error;
|
|
2721
|
-
}
|
|
2722
|
-
await this.createWatcher(id, watcherTurn);
|
|
2723
|
-
} finally {
|
|
2724
|
-
callTurn.release();
|
|
2725
|
-
watcherTurn.release();
|
|
2726
|
-
}
|
|
2727
|
-
const replayedResult = this.replay == null ? void 0 : await this.replay.takeResult(id);
|
|
2728
|
-
const result = replayedResult ?? await this.waitForLiveEvent(
|
|
2729
|
-
(entry) => entry.functionCallWatcherResult?.functionCallId === id
|
|
2730
|
-
).then((entry) => entry.functionCallWatcherResult);
|
|
2731
|
-
let resolved;
|
|
2732
|
-
try {
|
|
2733
|
-
resolved = await this.resolveWatcherResult(result);
|
|
2734
|
-
} finally {
|
|
2735
|
-
if (replayedResult != null) this.replay?.completeResultResolution();
|
|
2736
|
-
}
|
|
2737
|
-
this.log("info", "durable reduce completed", {
|
|
2738
|
-
durable_id: id,
|
|
2739
|
-
target_function: definition.name,
|
|
2740
|
-
collection_size: reduceItems.length,
|
|
2741
|
-
duration_ms: Date.now() - startedAt
|
|
2742
|
-
});
|
|
2743
|
-
return resolved;
|
|
2744
|
-
}
|
|
2745
|
-
nextDurableId(definition, operation = "FunctionCall") {
|
|
2746
|
-
const id = durableHash([
|
|
2747
|
-
String(this.allocation.functionCallId),
|
|
2748
|
-
this.previousDurableId,
|
|
2749
|
-
operation,
|
|
2750
|
-
definition.name
|
|
2751
|
-
]);
|
|
2752
|
-
this.previousDurableId = id;
|
|
2753
|
-
return id;
|
|
2754
|
-
}
|
|
2755
|
-
async runChild(definition, args, delaySeconds) {
|
|
2756
|
-
const callTurn = this.reserveEmissionTurn("call");
|
|
2757
|
-
const watcherTurn = this.reserveEmissionTurn("watcher");
|
|
2758
|
-
const id = this.nextDurableId(definition);
|
|
2759
|
-
const startedAt = Date.now();
|
|
2760
|
-
const maxRetries = definition.options.retries?.maxRetries ?? this.applicationDefinition.application?.retries.maxRetries ?? 0;
|
|
2761
|
-
this.log("info", "durable function call starting", {
|
|
2762
|
-
durable_id: id,
|
|
2763
|
-
target_function: definition.name,
|
|
2764
|
-
argument_count: args.length,
|
|
2765
|
-
delay_seconds: delaySeconds,
|
|
2766
|
-
max_retries: maxRetries
|
|
2767
|
-
});
|
|
2768
|
-
this.replay?.expectCall(id);
|
|
2769
|
-
this.replay?.expectWatcher(id);
|
|
2770
|
-
try {
|
|
2771
|
-
try {
|
|
2772
|
-
await this.createChildCall(id, definition, args, delaySeconds, callTurn);
|
|
2773
|
-
} catch (error) {
|
|
2774
|
-
this.replay?.abandonCall(id);
|
|
2775
|
-
this.replay?.abandonWatcher(id);
|
|
2776
|
-
throw error;
|
|
2777
|
-
}
|
|
2778
|
-
await this.createWatcher(id, watcherTurn);
|
|
2779
|
-
} finally {
|
|
2780
|
-
callTurn.release();
|
|
2781
|
-
watcherTurn.release();
|
|
2782
|
-
}
|
|
2783
|
-
this.log("debug", "waiting for terminal durable function result", {
|
|
2784
|
-
durable_id: id,
|
|
2785
|
-
target_function: definition.name,
|
|
2786
|
-
max_retries: maxRetries,
|
|
2787
|
-
replay_result_available: this.replay?.hasAvailableResult(id) ?? false
|
|
2788
|
-
});
|
|
2789
|
-
const replayedResult = this.replay == null ? void 0 : await this.replay.takeResult(id);
|
|
2790
|
-
const result = replayedResult ?? await this.waitForLiveEvent(
|
|
2791
|
-
(entry) => entry.functionCallWatcherResult?.functionCallId === id
|
|
2792
|
-
).then((entry) => entry.functionCallWatcherResult);
|
|
2793
|
-
let resolved;
|
|
2794
|
-
try {
|
|
2795
|
-
resolved = await this.resolveWatcherResult(result);
|
|
2796
|
-
} finally {
|
|
2797
|
-
if (replayedResult != null) this.replay?.completeResultResolution();
|
|
2798
|
-
}
|
|
2799
|
-
this.log("info", "durable function call completed", {
|
|
2800
|
-
durable_id: id,
|
|
2801
|
-
target_function: definition.name,
|
|
2802
|
-
max_retries: maxRetries,
|
|
2803
|
-
duration_ms: Date.now() - startedAt
|
|
2804
|
-
});
|
|
2805
|
-
return resolved;
|
|
2806
|
-
}
|
|
2807
|
-
async createChildCall(id, definition, args, delaySeconds, turn) {
|
|
2808
|
-
try {
|
|
2809
|
-
let turnAcquired = false;
|
|
2810
|
-
if (this.replay != null) {
|
|
2811
|
-
await turn.wait;
|
|
2812
|
-
turnAcquired = true;
|
|
2813
|
-
}
|
|
2814
|
-
const replayed = await this.replay?.takeOrdered("call", id);
|
|
2815
|
-
if (replayed != null) {
|
|
2816
|
-
this.log("debug", "replaying durable function creation", { durable_id: id });
|
|
2817
|
-
const error2 = statusError(replayed.status, "Failed to start function call");
|
|
2818
|
-
if (error2 != null) throw error2;
|
|
2819
|
-
return;
|
|
2820
|
-
}
|
|
2821
|
-
const prepared = [];
|
|
2822
|
-
let offset = 0;
|
|
2823
|
-
for (const arg of args) {
|
|
2824
|
-
const item = prepareSerializedObject(arg, offset);
|
|
2825
|
-
prepared.push(item);
|
|
2826
|
-
offset += item.bytes.byteLength;
|
|
2827
|
-
}
|
|
2828
|
-
let argsBlob;
|
|
2829
|
-
if (prepared.length > 0) {
|
|
2830
|
-
this.log("debug", "requesting durable function arguments blob", {
|
|
2831
|
-
durable_id: id,
|
|
2832
|
-
argument_count: args.length,
|
|
2833
|
-
serialized_bytes: offset
|
|
2834
|
-
});
|
|
2835
|
-
try {
|
|
2836
|
-
argsBlob = await this.requestOutputBlob(offset);
|
|
2837
|
-
argsBlob = await this.uploadBlob(
|
|
2838
|
-
argsBlob,
|
|
2839
|
-
joinPrepared(prepared),
|
|
2840
|
-
"durable function arguments",
|
|
2841
|
-
{ durable_id: id }
|
|
2842
|
-
);
|
|
2843
|
-
} catch (error2) {
|
|
2844
|
-
if (this.controller.signal.aborted) throw this.controller.signal.reason;
|
|
2845
|
-
throw this.latchProtocolError(
|
|
2846
|
-
`Failed to materialize durable function arguments for '${id}'`,
|
|
2847
|
-
error2
|
|
2848
|
-
);
|
|
2849
|
-
}
|
|
2850
|
-
}
|
|
2851
|
-
const metadata = Buffer.from(JSON.stringify({
|
|
2852
|
-
format: "tensorlake.typescript.function-call.v1",
|
|
2853
|
-
id,
|
|
2854
|
-
functionName: definition.name,
|
|
2855
|
-
argumentCount: args.length
|
|
2856
|
-
}), "utf8");
|
|
2857
|
-
const updates = {
|
|
2858
|
-
updates: [{ functionCall: {
|
|
2859
|
-
id,
|
|
2860
|
-
target: {
|
|
2861
|
-
namespace: this.functionRef.namespace,
|
|
2862
|
-
applicationName: this.functionRef.applicationName,
|
|
2863
|
-
applicationVersion: this.functionRef.applicationVersion,
|
|
2864
|
-
functionName: definition.name
|
|
2865
|
-
},
|
|
2866
|
-
args: prepared.map((item) => ({ value: item.object })),
|
|
2867
|
-
callMetadata: metadata
|
|
2868
|
-
} }],
|
|
2869
|
-
rootFunctionCallId: id
|
|
2870
|
-
};
|
|
2871
|
-
if (delaySeconds > 0) updates.startAt = protocolTimestamp(new Date(Date.now() + delaySeconds * 1e3));
|
|
2872
|
-
if (!turnAcquired) await turn.wait;
|
|
2873
|
-
this.log("debug", "queueing durable function creation event", {
|
|
2874
|
-
durable_id: id,
|
|
2875
|
-
target_function: definition.name,
|
|
2876
|
-
delay_seconds: delaySeconds
|
|
2877
|
-
});
|
|
2878
|
-
this.addBatch([{ createFunctionCall: { updates, argsBlob } }]);
|
|
2879
|
-
this.log("debug", "waiting for durable function creation acknowledgement", { durable_id: id });
|
|
2880
|
-
const createdPromise = this.waitForLiveEvent((entry) => entry.functionCallCreated?.functionCallId === id);
|
|
2881
|
-
turn.release();
|
|
2882
|
-
const created = await createdPromise;
|
|
2883
|
-
const error = statusError(created.functionCallCreated?.status, "Failed to start function call");
|
|
2884
|
-
if (error != null) throw error;
|
|
2885
|
-
} finally {
|
|
2886
|
-
turn.release();
|
|
2887
|
-
}
|
|
2888
|
-
}
|
|
2889
|
-
async createReduceChain(ids, definition, items, initial, turn) {
|
|
2890
|
-
try {
|
|
2891
|
-
let turnAcquired = false;
|
|
2892
|
-
if (this.replay != null) {
|
|
2893
|
-
await turn.wait;
|
|
2894
|
-
turnAcquired = true;
|
|
2895
|
-
}
|
|
2896
|
-
const livePlans = [];
|
|
2897
|
-
const liveRoots = [];
|
|
2898
|
-
for (let start = 0; start < items.length; start += MAX_REDUCE_CALLS_PER_PLAN) {
|
|
2899
|
-
const end = Math.min(items.length, start + MAX_REDUCE_CALLS_PER_PLAN);
|
|
2900
|
-
const rootId = ids[end - 1];
|
|
2901
|
-
const replayed = await this.replay?.takeOrdered("call", rootId);
|
|
2902
|
-
if (replayed != null) {
|
|
2903
|
-
this.log("debug", "replaying durable reduce plan creation", {
|
|
2904
|
-
durable_id: rootId,
|
|
2905
|
-
reduce_step_start: start,
|
|
2906
|
-
reduce_step_end: end
|
|
2907
|
-
});
|
|
2908
|
-
const error = statusError(replayed.status, "Failed to start reduce operation");
|
|
2909
|
-
if (error != null) throw error;
|
|
2910
|
-
continue;
|
|
2911
|
-
}
|
|
2912
|
-
const prepared = [];
|
|
2913
|
-
let offset = 0;
|
|
2914
|
-
if (start === 0) {
|
|
2915
|
-
const value = prepareSerializedObject(initial, offset);
|
|
2916
|
-
prepared.push(value);
|
|
2917
|
-
offset += value.bytes.byteLength;
|
|
2918
|
-
}
|
|
2919
|
-
for (let index = start; index < end; index += 1) {
|
|
2920
|
-
const value = prepareSerializedObject(items[index], offset);
|
|
2921
|
-
prepared.push(value);
|
|
2922
|
-
offset += value.bytes.byteLength;
|
|
2923
|
-
}
|
|
2924
|
-
let argsBlob;
|
|
2925
|
-
try {
|
|
2926
|
-
const requested = await this.requestOutputBlob(offset);
|
|
2927
|
-
argsBlob = await this.uploadBlob(
|
|
2928
|
-
requested,
|
|
2929
|
-
joinPrepared(prepared),
|
|
2930
|
-
"durable reduce arguments",
|
|
2931
|
-
{ durable_id: rootId, reduce_step_start: start, reduce_step_end: end }
|
|
2932
|
-
);
|
|
2933
|
-
} catch (error) {
|
|
2934
|
-
if (this.controller.signal.aborted) throw this.controller.signal.reason;
|
|
2935
|
-
throw this.latchProtocolError(
|
|
2936
|
-
`Failed to materialize durable reduce arguments for '${rootId}'`,
|
|
2937
|
-
error
|
|
2938
|
-
);
|
|
2939
|
-
}
|
|
2940
|
-
const itemOffset = start === 0 ? 1 : 0;
|
|
2941
|
-
const updates = [];
|
|
2942
|
-
for (let index = start; index < end; index += 1) {
|
|
2943
|
-
const id = ids[index];
|
|
2944
|
-
const accumulator = index === 0 ? { value: prepared[0].object } : { functionCallId: ids[index - 1] };
|
|
2945
|
-
const item = prepared[itemOffset + index - start];
|
|
2946
|
-
const metadata = Buffer.from(JSON.stringify({
|
|
2947
|
-
format: "tensorlake.typescript.function-call.v1",
|
|
2948
|
-
id,
|
|
2949
|
-
functionName: definition.name,
|
|
2950
|
-
argumentCount: 2,
|
|
2951
|
-
operation: "reduce",
|
|
2952
|
-
reduceRootId: ids[ids.length - 1],
|
|
2953
|
-
reduceStep: index,
|
|
2954
|
-
reduceStepCount: items.length
|
|
2955
|
-
}), "utf8");
|
|
2956
|
-
updates.push({ functionCall: {
|
|
2957
|
-
id,
|
|
2958
|
-
target: {
|
|
2959
|
-
namespace: this.functionRef.namespace,
|
|
2960
|
-
applicationName: this.functionRef.applicationName,
|
|
2961
|
-
applicationVersion: this.functionRef.applicationVersion,
|
|
2962
|
-
functionName: definition.name
|
|
2963
|
-
},
|
|
2964
|
-
args: [accumulator, { value: item.object }],
|
|
2965
|
-
callMetadata: metadata
|
|
2966
|
-
} });
|
|
2967
|
-
}
|
|
2968
|
-
livePlans.push({
|
|
2969
|
-
createFunctionCall: {
|
|
2970
|
-
updates: { updates, rootFunctionCallId: rootId },
|
|
2971
|
-
argsBlob
|
|
2972
|
-
}
|
|
2973
|
-
});
|
|
2974
|
-
liveRoots.push(rootId);
|
|
2975
|
-
}
|
|
2976
|
-
if (livePlans.length === 0) return;
|
|
2977
|
-
if (!turnAcquired) await turn.wait;
|
|
2978
|
-
this.log("debug", "queueing durable reduce function-call chains", {
|
|
2979
|
-
durable_id: ids[ids.length - 1],
|
|
2980
|
-
target_function: definition.name,
|
|
2981
|
-
reduce_step_count: items.length,
|
|
2982
|
-
plan_count: livePlans.length
|
|
2983
|
-
});
|
|
2984
|
-
this.addBatch(livePlans);
|
|
2985
|
-
const creationWaiters = liveRoots.map((rootId) => {
|
|
2986
|
-
const predicate = (entry) => entry.functionCallCreated?.functionCallId === rootId;
|
|
2987
|
-
return {
|
|
2988
|
-
predicate,
|
|
2989
|
-
promise: this.waitForLiveEvent(predicate)
|
|
2990
|
-
};
|
|
2991
|
-
});
|
|
2992
|
-
const createdPromises = creationWaiters.map(async ({ promise }) => {
|
|
2993
|
-
const created = await promise;
|
|
2994
|
-
const error = statusError(
|
|
2995
|
-
created.functionCallCreated?.status,
|
|
2996
|
-
"Failed to start reduce operation"
|
|
2997
|
-
);
|
|
2998
|
-
if (error != null) throw error;
|
|
2999
|
-
});
|
|
3000
|
-
turn.release();
|
|
3001
|
-
try {
|
|
3002
|
-
await Promise.all(createdPromises);
|
|
3003
|
-
} catch (error) {
|
|
3004
|
-
for (const { predicate } of creationWaiters) {
|
|
3005
|
-
const index = this.liveWaiters.findIndex(
|
|
3006
|
-
(waiter) => waiter.predicate === predicate
|
|
3007
|
-
);
|
|
3008
|
-
if (index >= 0) this.liveWaiters.splice(index, 1)[0].result.reject(error);
|
|
3009
|
-
}
|
|
3010
|
-
throw error;
|
|
3011
|
-
}
|
|
3012
|
-
} finally {
|
|
3013
|
-
turn.release();
|
|
3014
|
-
}
|
|
3015
|
-
}
|
|
3016
|
-
async createWatcher(id, turn) {
|
|
3017
|
-
try {
|
|
3018
|
-
await turn.wait;
|
|
3019
|
-
const replayed = await this.replay?.takeOrdered("watcher", id);
|
|
3020
|
-
if (replayed != null) {
|
|
3021
|
-
this.log("debug", "replaying durable function watcher creation", { durable_id: id });
|
|
3022
|
-
const error2 = statusError(replayed.status, "Failed to create function call watcher");
|
|
3023
|
-
if (error2 != null) throw error2;
|
|
3024
|
-
return;
|
|
3025
|
-
}
|
|
3026
|
-
this.log("debug", "queueing durable function watcher event", { durable_id: id });
|
|
3027
|
-
this.addBatch([{ createFunctionCallWatcher: { functionCallId: id } }]);
|
|
3028
|
-
this.log("debug", "waiting for durable function watcher acknowledgement", { durable_id: id });
|
|
3029
|
-
const createdPromise = this.waitForLiveEvent(
|
|
3030
|
-
(entry) => entry.functionCallWatcherCreated?.functionCallId === id
|
|
3031
|
-
);
|
|
3032
|
-
turn.release();
|
|
3033
|
-
const created = await createdPromise;
|
|
3034
|
-
const error = statusError(created.functionCallWatcherCreated?.status, "Failed to create function call watcher");
|
|
3035
|
-
if (error != null) throw error;
|
|
3036
|
-
} finally {
|
|
3037
|
-
turn.release();
|
|
3038
|
-
}
|
|
3039
|
-
}
|
|
3040
|
-
async resolveWatcherResult(event) {
|
|
3041
|
-
this.log("debug", "resolving durable function watcher result", {
|
|
3042
|
-
durable_id: event.functionCallId,
|
|
3043
|
-
watcher_status: event.watcherStatus,
|
|
3044
|
-
outcome_code: event.outcomeCode,
|
|
3045
|
-
has_value_output: event.valueOutput != null,
|
|
3046
|
-
has_request_error_output: event.requestErrorOutput != null
|
|
3047
|
-
});
|
|
3048
|
-
if (WATCHER_TIMED_OUT.has(event.watcherStatus)) throw new TimeoutError();
|
|
3049
|
-
if (OUTCOME_SUCCESS.has(event.outcomeCode)) {
|
|
3050
|
-
try {
|
|
3051
|
-
const downloaded = await this.downloadSerializedObject(
|
|
3052
|
-
event.valueOutput,
|
|
3053
|
-
event.valueBlob,
|
|
3054
|
-
"durable function result",
|
|
3055
|
-
{ durable_id: event.functionCallId }
|
|
3056
|
-
);
|
|
3057
|
-
return deserializeValueFromProtocol(downloaded);
|
|
3058
|
-
} catch (error) {
|
|
3059
|
-
if (this.controller.signal.aborted) throw this.controller.signal.reason;
|
|
3060
|
-
throw this.latchProtocolError(
|
|
3061
|
-
`Invalid successful function call watcher payload for '${String(event.functionCallId)}'`,
|
|
3062
|
-
error
|
|
3063
|
-
);
|
|
3064
|
-
}
|
|
3065
|
-
}
|
|
3066
|
-
if (OUTCOME_FAILURE.has(event.outcomeCode)) {
|
|
3067
|
-
if (event.requestErrorOutput != null) {
|
|
3068
|
-
let message;
|
|
3069
|
-
try {
|
|
3070
|
-
const downloaded = await this.downloadSerializedObject(
|
|
3071
|
-
event.requestErrorOutput,
|
|
3072
|
-
event.requestErrorBlob,
|
|
3073
|
-
"durable function request error",
|
|
3074
|
-
{ durable_id: event.functionCallId }
|
|
3075
|
-
);
|
|
3076
|
-
message = new TextDecoder("utf-8", { fatal: true }).decode(downloaded.data);
|
|
3077
|
-
} catch (error) {
|
|
3078
|
-
if (this.controller.signal.aborted) throw this.controller.signal.reason;
|
|
3079
|
-
throw this.latchProtocolError(
|
|
3080
|
-
`Invalid request-error function call watcher payload for '${String(event.functionCallId)}'`,
|
|
3081
|
-
error
|
|
3082
|
-
);
|
|
3083
|
-
}
|
|
3084
|
-
throw new RequestError(message);
|
|
3085
|
-
}
|
|
3086
|
-
throw new FunctionError("Function call failed");
|
|
3087
|
-
}
|
|
3088
|
-
throw this.latchProtocolError(
|
|
3089
|
-
`Unexpected function call watcher outcome '${String(event.outcomeCode)}'`
|
|
3090
|
-
);
|
|
3091
|
-
}
|
|
3092
|
-
latchProtocolError(message, cause) {
|
|
3093
|
-
const protocolError = this.protocolError ?? (message instanceof AllocationProtocolError ? message : new AllocationProtocolError(
|
|
3094
|
-
message,
|
|
3095
|
-
cause === void 0 ? void 0 : { cause }
|
|
3096
|
-
));
|
|
3097
|
-
this.protocolError = protocolError;
|
|
3098
|
-
this.abortExecution(protocolError);
|
|
3099
|
-
return protocolError;
|
|
3100
|
-
}
|
|
3101
|
-
async startTailCall(tailCall) {
|
|
3102
|
-
const callTurn = this.reserveEmissionTurn("call");
|
|
3103
|
-
const id = this.nextDurableId(tailCall.definition);
|
|
3104
|
-
this.log("info", "tail call starting", {
|
|
3105
|
-
durable_id: id,
|
|
3106
|
-
target_function: tailCall.definition.name,
|
|
3107
|
-
argument_count: tailCall.args.length
|
|
3108
|
-
});
|
|
3109
|
-
this.replay?.expectCall(id);
|
|
3110
|
-
try {
|
|
3111
|
-
await this.createChildCall(id, tailCall.definition, tailCall.args, 0, callTurn);
|
|
3112
|
-
} catch (error) {
|
|
3113
|
-
this.replay?.abandonCall(id);
|
|
3114
|
-
throw error;
|
|
3115
|
-
}
|
|
3116
|
-
return id;
|
|
3117
|
-
}
|
|
3118
|
-
reserveEmissionTurn(kind) {
|
|
3119
|
-
const previous = kind === "call" ? this.durableCallEmissionTail : this.durableWatcherEmissionTail;
|
|
3120
|
-
let signalRelease;
|
|
3121
|
-
const released = new Promise((resolve) => {
|
|
3122
|
-
signalRelease = resolve;
|
|
3123
|
-
});
|
|
3124
|
-
const tail = previous.then(() => released);
|
|
3125
|
-
if (kind === "call") this.durableCallEmissionTail = tail;
|
|
3126
|
-
else this.durableWatcherEmissionTail = tail;
|
|
3127
|
-
let didRelease = false;
|
|
3128
|
-
return {
|
|
3129
|
-
wait: previous,
|
|
3130
|
-
release: () => {
|
|
3131
|
-
if (didRelease) return;
|
|
3132
|
-
didRelease = true;
|
|
3133
|
-
signalRelease();
|
|
3134
|
-
}
|
|
3135
|
-
};
|
|
3136
|
-
}
|
|
3137
|
-
async run() {
|
|
3138
|
-
let handlerArgs;
|
|
3139
|
-
this.log("info", "allocation execution started", {
|
|
3140
|
-
replay_mode: this.allocation.replayMode,
|
|
3141
|
-
function: this.functionRef.functionName
|
|
3142
|
-
});
|
|
3143
|
-
this.emitLifecycleEvent("allocations_started", "Starting allocations");
|
|
3144
|
-
try {
|
|
3145
|
-
const inputs = this.allocation.inputs ?? {};
|
|
3146
|
-
const objects = inputs.args ?? [];
|
|
3147
|
-
const blobs = inputs.argBlobs ?? [];
|
|
3148
|
-
this.log("debug", "allocation inputs validated", {
|
|
3149
|
-
object_count: objects.length,
|
|
3150
|
-
blob_count: blobs.length,
|
|
3151
|
-
function_call_metadata_bytes: inputs.functionCallMetadata?.length ?? 0
|
|
3152
|
-
});
|
|
3153
|
-
if (objects.length !== blobs.length) throw new Error("Function argument and BLOB counts differ");
|
|
3154
|
-
const values = await Promise.all(objects.map(
|
|
3155
|
-
(object, index) => this.downloadSerializedObject(
|
|
3156
|
-
object,
|
|
3157
|
-
blobs[index],
|
|
3158
|
-
"function input",
|
|
3159
|
-
{ input_index: index }
|
|
3160
|
-
)
|
|
3161
|
-
));
|
|
3162
|
-
this.log("info", "all function inputs downloaded", {
|
|
3163
|
-
input_count: values.length,
|
|
3164
|
-
total_data_bytes: values.reduce((total, value) => total + value.data.byteLength, 0)
|
|
3165
|
-
});
|
|
3166
|
-
const metadata = Buffer.from(inputs.functionCallMetadata ?? []);
|
|
3167
|
-
if (metadata.byteLength === 0) {
|
|
3168
|
-
this.log("debug", "deserializing application invocation input", {
|
|
3169
|
-
content_type: values[0]?.contentType
|
|
3170
|
-
});
|
|
3171
|
-
if (values.length !== 1) throw new Error("Application calls require exactly one input payload");
|
|
3172
|
-
try {
|
|
3173
|
-
const parsed = await deserializeApplicationArguments(this.definition, values[0]);
|
|
3174
|
-
handlerArgs = parsed.args;
|
|
3175
|
-
} catch (error) {
|
|
3176
|
-
this.log("error", "application invocation input deserialization failed", {}, error);
|
|
3177
|
-
await this.finishUserError(error);
|
|
3178
|
-
return;
|
|
3179
|
-
}
|
|
3180
|
-
this.log("debug", "application invocation input deserialized", {
|
|
3181
|
-
handler_argument_count: handlerArgs.length
|
|
3182
|
-
});
|
|
3183
|
-
} else {
|
|
3184
|
-
this.log("debug", "deserializing internal function invocation inputs", {
|
|
3185
|
-
metadata_bytes: metadata.byteLength
|
|
3186
|
-
});
|
|
3187
|
-
const call = JSON.parse(metadata.toString("utf8"));
|
|
3188
|
-
if (call.format !== "tensorlake.typescript.function-call.v1") {
|
|
3189
|
-
throw new Error("Function call metadata was produced by a different SDK language");
|
|
3190
|
-
}
|
|
3191
|
-
if (call.argumentCount !== values.length) throw new Error("Function call metadata argument count differs");
|
|
3192
|
-
try {
|
|
3193
|
-
handlerArgs = values.map(deserializeValueFromProtocol);
|
|
3194
|
-
} catch (error) {
|
|
3195
|
-
this.log("error", "internal function argument deserialization failed", {}, error);
|
|
3196
|
-
await this.finishUserError(error);
|
|
3197
|
-
return;
|
|
3198
|
-
}
|
|
3199
|
-
this.log("debug", "internal function invocation inputs deserialized", {
|
|
3200
|
-
handler_argument_count: handlerArgs.length,
|
|
3201
|
-
metadata_function: call.functionName
|
|
3202
|
-
});
|
|
3203
|
-
}
|
|
3204
|
-
if (REPLAY_STRICT.has(this.allocation.replayMode)) {
|
|
3205
|
-
this.log("info", "strict replay history loading");
|
|
3206
|
-
await this.loadReplayHistory();
|
|
3207
|
-
} else {
|
|
3208
|
-
this.log("debug", "strict replay disabled");
|
|
3209
|
-
}
|
|
3210
|
-
const requestContext = this.createRequestContext();
|
|
3211
|
-
let result;
|
|
3212
|
-
const handlerStartedAt = Date.now();
|
|
3213
|
-
this.log("info", "user function handler starting", {
|
|
3214
|
-
handler_argument_count: handlerArgs.length
|
|
3215
|
-
});
|
|
3216
|
-
try {
|
|
3217
|
-
const execute = () => executeHandlerResult(this.definition, handlerArgs);
|
|
3218
|
-
result = await runWithRequestContext(
|
|
3219
|
-
requestContext,
|
|
3220
|
-
() => runWithFunctionRuntime(this, () => waitWithAbortSignal(
|
|
3221
|
-
this.replay?.runCausalHandler(execute) ?? execute(),
|
|
3222
|
-
this.controller.signal
|
|
3223
|
-
))
|
|
3224
|
-
);
|
|
3225
|
-
} catch (error) {
|
|
3226
|
-
if (error instanceof ReplayMismatchError || error instanceof AllocationProtocolError) {
|
|
3227
|
-
throw error;
|
|
3228
|
-
}
|
|
3229
|
-
this.replay?.assertNoMismatch();
|
|
3230
|
-
this.log("error", "user function handler failed", {
|
|
3231
|
-
duration_ms: Date.now() - handlerStartedAt
|
|
3232
|
-
}, error);
|
|
3233
|
-
await this.finishUserError(error);
|
|
3234
|
-
return;
|
|
3235
|
-
}
|
|
3236
|
-
this.log("info", "user function handler completed", {
|
|
3237
|
-
duration_ms: Date.now() - handlerStartedAt,
|
|
3238
|
-
result_kind: isTailCall(result) ? "tail_call" : "value",
|
|
3239
|
-
result_type: result == null ? String(result) : typeof result
|
|
3240
|
-
});
|
|
3241
|
-
if (this.protocolError != null) throw this.protocolError;
|
|
3242
|
-
this.replay?.assertNoMismatch();
|
|
3243
|
-
if (isTailCall(result)) {
|
|
3244
|
-
const id = await this.startTailCall(result);
|
|
3245
|
-
this.replay?.assertConsumed();
|
|
3246
|
-
this.controller.signal.throwIfAborted();
|
|
3247
|
-
this.log("info", "queueing successful tail-call allocation finish", { tail_call_durable_id: id });
|
|
3248
|
-
this.queueTerminalBatch({
|
|
3249
|
-
outcomeCode: "ALLOCATION_OUTCOME_CODE_SUCCESS",
|
|
3250
|
-
tailCallDurableId: id
|
|
3251
|
-
});
|
|
3252
|
-
} else {
|
|
3253
|
-
this.replay?.assertConsumed();
|
|
3254
|
-
let prepared;
|
|
3255
|
-
try {
|
|
3256
|
-
prepared = prepareSerializedObject(result, 0, String(this.allocation.functionCallId));
|
|
3257
|
-
} catch (error) {
|
|
3258
|
-
this.log("error", "function output serialization failed", {}, error);
|
|
3259
|
-
await this.finishUserError(error);
|
|
3260
|
-
return;
|
|
3261
|
-
}
|
|
3262
|
-
this.log("debug", "function output serialized", {
|
|
3263
|
-
output_bytes: prepared.bytes.byteLength,
|
|
3264
|
-
output_encoding: prepared.object.manifest?.encoding,
|
|
3265
|
-
output_content_type: prepared.object.manifest?.contentType
|
|
3266
|
-
});
|
|
3267
|
-
let outputBlob = await this.requestOutputBlob(prepared.bytes.byteLength);
|
|
3268
|
-
outputBlob = await this.uploadBlob(outputBlob, prepared.bytes, "function output");
|
|
3269
|
-
this.controller.signal.throwIfAborted();
|
|
3270
|
-
this.log("info", "queueing successful value allocation finish", {
|
|
3271
|
-
output_blob_id: outputBlob.id,
|
|
3272
|
-
output_bytes: prepared.bytes.byteLength
|
|
3273
|
-
});
|
|
3274
|
-
this.queueTerminalBatch({
|
|
3275
|
-
outcomeCode: "ALLOCATION_OUTCOME_CODE_SUCCESS",
|
|
3276
|
-
value: prepared.object,
|
|
3277
|
-
uploadedFunctionOutputsBlob: outputBlob
|
|
3278
|
-
});
|
|
3279
|
-
}
|
|
3280
|
-
} catch (error) {
|
|
3281
|
-
const replayMismatch = error instanceof ReplayMismatchError;
|
|
3282
|
-
const protocolFailure = error instanceof AllocationProtocolError || this.protocolError != null;
|
|
3283
|
-
const cancelled = this.controller.signal.aborted && !protocolFailure;
|
|
3284
|
-
this.log(
|
|
3285
|
-
cancelled ? "info" : "error",
|
|
3286
|
-
cancelled ? "allocation stopped after cancellation" : replayMismatch ? "durable replay mismatch" : "allocation internal failure",
|
|
3287
|
-
{},
|
|
3288
|
-
error
|
|
3289
|
-
);
|
|
3290
|
-
this.queueTerminalBatch({
|
|
3291
|
-
outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
|
|
3292
|
-
failureReason: replayMismatch ? "ALLOCATION_FAILURE_REASON_REPLAY_EVENT_HISTORY_MISMATCH" : protocolFailure ? "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR" : cancelled ? "ALLOCATION_FAILURE_REASON_FUNCTION_ERROR" : "ALLOCATION_FAILURE_REASON_INTERNAL_ERROR"
|
|
3293
|
-
});
|
|
3294
|
-
} finally {
|
|
3295
|
-
this.log("info", "allocation execution stopping", {
|
|
3296
|
-
duration_ms: Date.now() - this.startedAt,
|
|
3297
|
-
pending_output_blob_requests: this.outputBlobRequests.size,
|
|
3298
|
-
pending_state_operations: this.stateOperationRequests.size,
|
|
3299
|
-
pending_event_waiters: this.liveWaiters.length
|
|
3300
|
-
});
|
|
3301
|
-
this.emitLifecycleEvent("allocations_finished", "Allocations completed");
|
|
3302
|
-
this.finish();
|
|
3303
|
-
this.replay?.dispose();
|
|
3304
|
-
}
|
|
3305
|
-
}
|
|
3306
|
-
async finishUserError(error) {
|
|
3307
|
-
const rendered = error instanceof Error ? error.message : String(error);
|
|
3308
|
-
this.runWithoutReplayOwnership(() => {
|
|
3309
|
-
printCloudEvent({
|
|
3310
|
-
level: "error",
|
|
3311
|
-
event: "function_call_failed",
|
|
3312
|
-
message: rendered,
|
|
3313
|
-
namespace: this.functionRef.namespace,
|
|
3314
|
-
application: this.functionRef.applicationName,
|
|
3315
|
-
application_version: this.functionRef.applicationVersion,
|
|
3316
|
-
function: this.functionRef.functionName,
|
|
3317
|
-
request_id: this.allocation.requestId,
|
|
3318
|
-
function_call_id: this.allocation.functionCallId,
|
|
3319
|
-
allocation_id: this.allocation.allocationId,
|
|
3320
|
-
error: error instanceof Error ? error.stack?.split("\n") ?? [rendered] : [rendered]
|
|
3321
|
-
});
|
|
3322
|
-
});
|
|
3323
|
-
if (isRequestError(error)) {
|
|
3324
|
-
const output = prepareTextObject(error.message);
|
|
3325
|
-
const requestErrorBlob = this.allocation.inputs?.requestErrorBlob;
|
|
3326
|
-
if (requestErrorBlob == null) throw new Error("Allocation has no request error BLOB");
|
|
3327
|
-
const uploaded = await this.uploadBlob(requestErrorBlob, output.bytes, "request error output");
|
|
3328
|
-
this.log("info", "queueing request-error allocation finish", {
|
|
3329
|
-
request_error_bytes: output.bytes.byteLength,
|
|
3330
|
-
request_error_blob_id: uploaded.id
|
|
3331
|
-
});
|
|
3332
|
-
this.queueTerminalBatch({
|
|
3333
|
-
outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
|
|
3334
|
-
failureReason: "ALLOCATION_FAILURE_REASON_REQUEST_ERROR",
|
|
3335
|
-
requestErrorOutput: output.object,
|
|
3336
|
-
uploadedRequestErrorBlob: uploaded
|
|
3337
|
-
});
|
|
3338
|
-
return;
|
|
3339
|
-
}
|
|
3340
|
-
this.log("info", "queueing function-error allocation finish", {}, error);
|
|
3341
|
-
this.queueTerminalBatch({
|
|
3342
|
-
outcomeCode: "ALLOCATION_OUTCOME_CODE_FAILURE",
|
|
3343
|
-
failureReason: "ALLOCATION_FAILURE_REASON_FUNCTION_ERROR"
|
|
3344
|
-
});
|
|
3345
|
-
}
|
|
3346
|
-
createRequestContext() {
|
|
3347
|
-
return {
|
|
3348
|
-
requestId: String(this.allocation.requestId),
|
|
3349
|
-
headers: new Headers2(
|
|
3350
|
-
(this.allocation.inputs?.requestContext?.headers ?? []).map(
|
|
3351
|
-
(header) => [
|
|
3352
|
-
String(header.name ?? ""),
|
|
3353
|
-
String(header.value ?? "")
|
|
3354
|
-
]
|
|
3355
|
-
)
|
|
3356
|
-
),
|
|
3357
|
-
signal: this.controller.signal,
|
|
3358
|
-
state: {
|
|
3359
|
-
get: async (key, defaultValue) => {
|
|
3360
|
-
validateRequestStateKey(key);
|
|
3361
|
-
const operationId = (0, import_node_crypto5.randomUUID)();
|
|
3362
|
-
this.log("debug", "request state read starting", {
|
|
3363
|
-
state_key: key,
|
|
3364
|
-
operation_id: operationId,
|
|
3365
|
-
has_default: defaultValue !== void 0
|
|
3366
|
-
});
|
|
3367
|
-
const result = await this.requestStateOperation({ operationId, stateKey: key, prepareRead: {} });
|
|
3368
|
-
if (result.status?.code === NOT_FOUND) {
|
|
3369
|
-
this.log("debug", "request state key not found", { state_key: key, operation_id: operationId });
|
|
3370
|
-
return defaultValue;
|
|
3371
|
-
}
|
|
3372
|
-
const error = statusError(result.status, `Failed to read request state '${key}'`);
|
|
3373
|
-
if (error != null) throw error;
|
|
3374
|
-
const data = await this.downloadBlob(result.prepareRead?.blob, "request state value", {
|
|
3375
|
-
state_key: key,
|
|
3376
|
-
operation_id: operationId
|
|
3377
|
-
});
|
|
3378
|
-
const value = deserializeJSON(data);
|
|
3379
|
-
this.log("debug", "request state read completed", {
|
|
3380
|
-
state_key: key,
|
|
3381
|
-
operation_id: operationId,
|
|
3382
|
-
value_bytes: data.byteLength
|
|
3383
|
-
});
|
|
3384
|
-
return value;
|
|
3385
|
-
},
|
|
3386
|
-
set: async (key, value) => {
|
|
3387
|
-
validateRequestStateKey(key);
|
|
3388
|
-
const serialized = serializeRequestStateValue(value);
|
|
3389
|
-
const prepareOperationId = (0, import_node_crypto5.randomUUID)();
|
|
3390
|
-
this.log("debug", "request state write preparation starting", {
|
|
3391
|
-
state_key: key,
|
|
3392
|
-
operation_id: prepareOperationId,
|
|
3393
|
-
value_bytes: serialized.byteLength
|
|
3394
|
-
});
|
|
3395
|
-
const prepared = await this.requestStateOperation({
|
|
3396
|
-
operationId: prepareOperationId,
|
|
3397
|
-
stateKey: key,
|
|
3398
|
-
prepareWrite: { size: serialized.byteLength }
|
|
3399
|
-
});
|
|
3400
|
-
const error = statusError(prepared.status, `Failed to prepare request state '${key}'`);
|
|
3401
|
-
if (error != null) throw error;
|
|
3402
|
-
const blob = await this.uploadBlob(prepared.prepareWrite?.blob, serialized, "request state value", {
|
|
3403
|
-
state_key: key,
|
|
3404
|
-
operation_id: prepareOperationId
|
|
3405
|
-
});
|
|
3406
|
-
const commitOperationId = (0, import_node_crypto5.randomUUID)();
|
|
3407
|
-
this.log("debug", "request state write commit starting", {
|
|
3408
|
-
state_key: key,
|
|
3409
|
-
operation_id: commitOperationId,
|
|
3410
|
-
blob_id: blob.id
|
|
3411
|
-
});
|
|
3412
|
-
const committed = await this.requestStateOperation({
|
|
3413
|
-
operationId: commitOperationId,
|
|
3414
|
-
stateKey: key,
|
|
3415
|
-
commitWrite: { blob }
|
|
3416
|
-
});
|
|
3417
|
-
const commitError = statusError(committed.status, `Failed to commit request state '${key}'`);
|
|
3418
|
-
if (commitError != null) throw commitError;
|
|
3419
|
-
this.log("debug", "request state write completed", {
|
|
3420
|
-
state_key: key,
|
|
3421
|
-
operation_id: commitOperationId,
|
|
3422
|
-
value_bytes: serialized.byteLength
|
|
3423
|
-
});
|
|
3424
|
-
}
|
|
3425
|
-
},
|
|
3426
|
-
metrics: {
|
|
3427
|
-
counter: async (name, value = 1) => {
|
|
3428
|
-
validateCounterMetric(name, value);
|
|
3429
|
-
this.emitUserEvent("ai.tensorlake.metric.counter.inc", {
|
|
3430
|
-
counter_name: name,
|
|
3431
|
-
counter_inc: value
|
|
3432
|
-
});
|
|
3433
|
-
},
|
|
3434
|
-
timer: async (name, value) => {
|
|
3435
|
-
validateTimerMetric(name, value);
|
|
3436
|
-
this.emitUserEvent("ai.tensorlake.metric.timer", {
|
|
3437
|
-
timer_name: name,
|
|
3438
|
-
timer_value: value
|
|
3439
|
-
});
|
|
3440
|
-
}
|
|
3441
|
-
},
|
|
3442
|
-
progress: {
|
|
3443
|
-
update: async (current, total, options) => {
|
|
3444
|
-
validateProgressUpdate(current, total, options);
|
|
3445
|
-
this.log("debug", "request progress update", {
|
|
3446
|
-
current,
|
|
3447
|
-
total,
|
|
3448
|
-
has_custom_message: options?.message != null,
|
|
3449
|
-
attribute_count: Object.keys(options?.attributes ?? {}).length
|
|
3450
|
-
});
|
|
3451
|
-
this.state.progress = { current, total };
|
|
3452
|
-
this.publishState();
|
|
3453
|
-
const message = options?.message ?? `${this.functionRef.functionName}: executing step ${current} of ${total}`;
|
|
3454
|
-
this.runWithoutReplayOwnership(() => {
|
|
3455
|
-
printCloudEvent({ RequestProgressUpdated: {
|
|
3456
|
-
request_id: this.allocation.requestId,
|
|
3457
|
-
function_name: this.functionRef.functionName,
|
|
3458
|
-
function_run_id: this.allocation.functionCallId,
|
|
3459
|
-
allocation_id: this.allocation.allocationId,
|
|
3460
|
-
message,
|
|
3461
|
-
step: current,
|
|
3462
|
-
total,
|
|
3463
|
-
attributes: options?.attributes,
|
|
3464
|
-
created_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
3465
|
-
} }, {
|
|
3466
|
-
type: "ai.tensorlake.progress_update",
|
|
3467
|
-
source: "/tensorlake/applications/progress",
|
|
3468
|
-
message
|
|
3469
|
-
});
|
|
3470
|
-
});
|
|
3471
|
-
}
|
|
3472
|
-
}
|
|
3473
|
-
};
|
|
3474
|
-
}
|
|
3475
|
-
emitUserEvent(type, data) {
|
|
3476
|
-
this.log("debug", "user metric event emitted", {
|
|
3477
|
-
event_type: type,
|
|
3478
|
-
metric_name: data.counter_name ?? data.timer_name
|
|
3479
|
-
});
|
|
3480
|
-
this.runWithoutReplayOwnership(() => {
|
|
3481
|
-
printCloudEvent({
|
|
3482
|
-
request_id: this.allocation.requestId,
|
|
3483
|
-
function_name: this.functionRef.functionName,
|
|
3484
|
-
function_run_id: this.allocation.functionCallId,
|
|
3485
|
-
allocation_id: this.allocation.allocationId,
|
|
3486
|
-
...data
|
|
3487
|
-
}, { type, source: "/tensorlake/applications/metrics" });
|
|
3488
|
-
});
|
|
3489
|
-
}
|
|
3490
|
-
emitLifecycleEvent(event, message) {
|
|
3491
|
-
this.runWithoutReplayOwnership(() => {
|
|
3492
|
-
printCloudEvent({
|
|
3493
|
-
event,
|
|
3494
|
-
message,
|
|
3495
|
-
allocations: [{
|
|
3496
|
-
namespace: this.functionRef.namespace,
|
|
3497
|
-
application: this.functionRef.applicationName,
|
|
3498
|
-
application_version: this.functionRef.applicationVersion,
|
|
3499
|
-
function: this.functionRef.functionName,
|
|
3500
|
-
request_id: this.allocation.requestId,
|
|
3501
|
-
function_call_id: this.allocation.functionCallId,
|
|
3502
|
-
allocation_id: this.allocation.allocationId
|
|
3503
|
-
}]
|
|
3504
|
-
});
|
|
3505
|
-
});
|
|
3506
|
-
}
|
|
3507
|
-
async loadReplayHistory() {
|
|
3508
|
-
const entries = [];
|
|
3509
|
-
let afterClock = 0;
|
|
3510
|
-
let page = 0;
|
|
3511
|
-
while (true) {
|
|
3512
|
-
page += 1;
|
|
3513
|
-
this.log("debug", "requesting replay history page", { page, after_clock: afterClock });
|
|
3514
|
-
const requestedAfterClock = afterClock;
|
|
3515
|
-
const response = await this.readEvents(afterClock);
|
|
3516
|
-
entries.push(...response.entries ?? []);
|
|
3517
|
-
this.log("debug", "replay history page received", {
|
|
3518
|
-
page,
|
|
3519
|
-
entry_count: response.entries?.length ?? 0,
|
|
3520
|
-
last_clock: response.lastClock,
|
|
3521
|
-
has_more: response.hasMore
|
|
3522
|
-
});
|
|
3523
|
-
const responseClock = validatedEventLogClock(
|
|
3524
|
-
response,
|
|
3525
|
-
requestedAfterClock,
|
|
3526
|
-
(message) => new ReplayMismatchError(message)
|
|
3527
|
-
);
|
|
3528
|
-
afterClock = responseClock;
|
|
3529
|
-
if (!response.hasMore) break;
|
|
3530
|
-
}
|
|
3531
|
-
this.lastEventClock = afterClock;
|
|
3532
|
-
this.replay = new ReplayHistory(entries);
|
|
3533
|
-
this.log("info", "strict replay history loaded", {
|
|
3534
|
-
page_count: page,
|
|
3535
|
-
entry_count: entries.length,
|
|
3536
|
-
last_clock: afterClock
|
|
3537
|
-
});
|
|
3538
|
-
}
|
|
3539
|
-
waitForLiveEvent(predicate) {
|
|
3540
|
-
if (this.controller.signal.aborted) {
|
|
3541
|
-
return Promise.reject(this.controller.signal.reason);
|
|
3542
|
-
}
|
|
3543
|
-
const index = this.liveBacklog.findIndex(predicate);
|
|
3544
|
-
if (index >= 0) {
|
|
3545
|
-
this.log("debug", "durable event satisfied from backlog", {
|
|
3546
|
-
backlog_size: this.liveBacklog.length,
|
|
3547
|
-
pending_waiters: this.liveWaiters.length
|
|
3548
|
-
});
|
|
3549
|
-
return Promise.resolve(this.liveBacklog.splice(index, 1)[0]);
|
|
3550
|
-
}
|
|
3551
|
-
const result = deferred();
|
|
3552
|
-
this.liveWaiters.push({ predicate, result });
|
|
3553
|
-
this.log("debug", "waiting for durable event", {
|
|
3554
|
-
backlog_size: this.liveBacklog.length,
|
|
3555
|
-
pending_waiters: this.liveWaiters.length
|
|
3556
|
-
});
|
|
3557
|
-
this.ensureLivePump();
|
|
3558
|
-
return result.promise;
|
|
3559
|
-
}
|
|
3560
|
-
ensureLivePump() {
|
|
3561
|
-
if (this.livePump != null) {
|
|
3562
|
-
this.log("debug", "durable event live pump already running");
|
|
3563
|
-
return;
|
|
3564
|
-
}
|
|
3565
|
-
this.log("info", "durable event live pump starting", { after_clock: this.lastEventClock });
|
|
3566
|
-
const pump = (async () => {
|
|
3567
|
-
while (!this.finished && this.liveWaiters.length > 0) {
|
|
3568
|
-
const requestedAfterClock = this.lastEventClock;
|
|
3569
|
-
const response = await this.readEvents(this.lastEventClock);
|
|
3570
|
-
this.log("debug", "durable event log page received", {
|
|
3571
|
-
requested_after_clock: requestedAfterClock,
|
|
3572
|
-
entry_count: response.entries?.length ?? 0,
|
|
3573
|
-
last_clock: response.lastClock,
|
|
3574
|
-
has_more: response.hasMore
|
|
3575
|
-
});
|
|
3576
|
-
const responseClock = validatedEventLogClock(
|
|
3577
|
-
response,
|
|
3578
|
-
requestedAfterClock,
|
|
3579
|
-
(message) => new AllocationProtocolError(message)
|
|
3580
|
-
);
|
|
3581
|
-
this.lastEventClock = responseClock;
|
|
3582
|
-
for (const entry of response.entries ?? []) this.dispatchLiveEvent(entry);
|
|
3583
|
-
}
|
|
3584
|
-
})();
|
|
3585
|
-
this.livePump = pump;
|
|
3586
|
-
void pump.then(() => {
|
|
3587
|
-
if (this.livePump !== pump) return;
|
|
3588
|
-
this.livePump = void 0;
|
|
3589
|
-
this.log("debug", "durable event live pump stopped", {
|
|
3590
|
-
finished: this.finished,
|
|
3591
|
-
pending_waiters: this.liveWaiters.length,
|
|
3592
|
-
after_clock: this.lastEventClock
|
|
3593
|
-
});
|
|
3594
|
-
if (!this.finished && this.liveWaiters.length > 0) this.ensureLivePump();
|
|
3595
|
-
}, (error) => {
|
|
3596
|
-
if (this.livePump === pump) this.livePump = void 0;
|
|
3597
|
-
if (!this.finished && !this.controller.signal.aborted) {
|
|
3598
|
-
const protocolError = error instanceof AllocationProtocolError ? error : new AllocationProtocolError(
|
|
3599
|
-
error instanceof Error ? error.message : String(error),
|
|
3600
|
-
{ cause: error }
|
|
3601
|
-
);
|
|
3602
|
-
this.latchProtocolError(protocolError);
|
|
3603
|
-
this.log("error", "durable event live pump failed", {
|
|
3604
|
-
pending_waiters: this.liveWaiters.length
|
|
3605
|
-
}, protocolError);
|
|
3606
|
-
}
|
|
3607
|
-
});
|
|
3608
|
-
}
|
|
3609
|
-
dispatchLiveEvent(entry) {
|
|
3610
|
-
const index = this.liveWaiters.findIndex((waiter) => waiter.predicate(entry));
|
|
3611
|
-
const eventKind = Object.keys(entry).find((key) => key !== "clock") ?? "unknown";
|
|
3612
|
-
if (index < 0) {
|
|
3613
|
-
this.liveBacklog.push(entry);
|
|
3614
|
-
this.log("debug", "durable event added to backlog", {
|
|
3615
|
-
event_kind: eventKind,
|
|
3616
|
-
event_clock: entry.clock,
|
|
3617
|
-
backlog_size: this.liveBacklog.length
|
|
3618
|
-
});
|
|
3619
|
-
} else {
|
|
3620
|
-
this.liveWaiters.splice(index, 1)[0].result.resolve(entry);
|
|
3621
|
-
this.log("debug", "durable event delivered to waiter", {
|
|
3622
|
-
event_kind: eventKind,
|
|
3623
|
-
event_clock: entry.clock,
|
|
3624
|
-
pending_waiters: this.liveWaiters.length
|
|
3625
|
-
});
|
|
3626
|
-
}
|
|
3627
|
-
}
|
|
3628
|
-
readEvents(afterClock) {
|
|
3629
|
-
if (this.currentRead != null) {
|
|
3630
|
-
throw new AllocationProtocolError("Allocation event log already has a pending read");
|
|
3631
|
-
}
|
|
3632
|
-
const response = deferred();
|
|
3633
|
-
const request = {
|
|
3634
|
-
allocationId: this.allocation.allocationId,
|
|
3635
|
-
afterClock,
|
|
3636
|
-
maxEntries: 100
|
|
3637
|
-
};
|
|
3638
|
-
this.currentRead = { request, response };
|
|
3639
|
-
this.log("debug", "event log read requested", {
|
|
3640
|
-
after_clock: afterClock,
|
|
3641
|
-
max_entries: request.maxEntries,
|
|
3642
|
-
connected_streams: this.eventReadStreams.size
|
|
3643
|
-
});
|
|
3644
|
-
for (const stream of [...this.eventReadStreams]) {
|
|
3645
|
-
try {
|
|
3646
|
-
stream.write(request);
|
|
3647
|
-
} catch (error) {
|
|
3648
|
-
this.dropEventReadStream(stream, "write_failed", error);
|
|
3649
|
-
}
|
|
3650
|
-
}
|
|
3651
|
-
return response.promise;
|
|
3652
|
-
}
|
|
3653
|
-
watchEventLogReads(stream) {
|
|
3654
|
-
this.eventReadStreams.add(stream);
|
|
3655
|
-
this.log("info", "event log read stream connected", {
|
|
3656
|
-
connected_streams: this.eventReadStreams.size,
|
|
3657
|
-
has_pending_read: this.currentRead != null
|
|
3658
|
-
});
|
|
3659
|
-
try {
|
|
3660
|
-
stream.on("cancelled", () => this.dropEventReadStream(stream, "cancelled"));
|
|
3661
|
-
stream.on("close", () => this.dropEventReadStream(stream, "closed"));
|
|
3662
|
-
if (this.currentRead != null) {
|
|
3663
|
-
this.log("debug", "sending pending event log read to new stream", {
|
|
3664
|
-
after_clock: this.currentRead.request.afterClock,
|
|
3665
|
-
max_entries: this.currentRead.request.maxEntries
|
|
3666
|
-
});
|
|
3667
|
-
stream.write(this.currentRead.request);
|
|
3668
|
-
}
|
|
3669
|
-
if (this.finished && this.currentRead == null) stream.end();
|
|
3670
|
-
} catch (error) {
|
|
3671
|
-
this.dropEventReadStream(stream, "setup_failed", error);
|
|
3672
|
-
throw error;
|
|
3673
|
-
}
|
|
3674
|
-
}
|
|
3675
|
-
deliverEventLogResponse(response) {
|
|
3676
|
-
if (this.currentRead == null) {
|
|
3677
|
-
this.log("warn", "unexpected event log response ignored", {
|
|
3678
|
-
entry_count: response.entries?.length ?? 0,
|
|
3679
|
-
last_clock: response.lastClock
|
|
3680
|
-
});
|
|
3681
|
-
return;
|
|
3682
|
-
}
|
|
3683
|
-
const pending = this.currentRead;
|
|
3684
|
-
this.currentRead = void 0;
|
|
3685
|
-
this.log("debug", "event log response delivered", {
|
|
3686
|
-
requested_after_clock: pending.request.afterClock,
|
|
3687
|
-
entry_count: response.entries?.length ?? 0,
|
|
3688
|
-
last_clock: response.lastClock,
|
|
3689
|
-
has_more: response.hasMore
|
|
3690
|
-
});
|
|
3691
|
-
pending.response.resolve(response);
|
|
3692
|
-
}
|
|
3693
|
-
requestOutputBlob(size) {
|
|
3694
|
-
if (this.controller.signal.aborted) {
|
|
3695
|
-
return Promise.reject(this.controller.signal.reason);
|
|
3696
|
-
}
|
|
3697
|
-
const id = (0, import_node_crypto5.randomUUID)();
|
|
3698
|
-
const request = deferred();
|
|
3699
|
-
this.outputBlobRequests.set(id, request);
|
|
3700
|
-
this.state.outputBlobRequests.push({ id, size });
|
|
3701
|
-
this.log("debug", "output blob requested", {
|
|
3702
|
-
blob_id: id,
|
|
3703
|
-
requested_bytes: size,
|
|
3704
|
-
pending_output_blob_requests: this.outputBlobRequests.size
|
|
3705
|
-
});
|
|
3706
|
-
this.publishState();
|
|
3707
|
-
return request.promise.then((output) => {
|
|
3708
|
-
this.log("debug", "output blob response received", {
|
|
3709
|
-
blob_id: id,
|
|
3710
|
-
status_code: output.status?.code,
|
|
3711
|
-
chunk_count: output.blob?.chunks?.length ?? 0
|
|
3712
|
-
});
|
|
3713
|
-
const error = statusError(output.status, "Failed to create output BLOB");
|
|
3714
|
-
if (error != null) throw error;
|
|
3715
|
-
return output.blob;
|
|
3716
|
-
});
|
|
3717
|
-
}
|
|
3718
|
-
requestStateOperation(operation) {
|
|
3719
|
-
if (this.controller.signal.aborted) {
|
|
3720
|
-
return Promise.reject(this.controller.signal.reason);
|
|
3721
|
-
}
|
|
3722
|
-
const request = deferred();
|
|
3723
|
-
this.stateOperationRequests.set(operation.operationId, request);
|
|
3724
|
-
this.state.requestStateOperations.push(operation);
|
|
3725
|
-
this.log("debug", "request state operation queued", {
|
|
3726
|
-
operation_id: operation.operationId,
|
|
3727
|
-
state_key: operation.stateKey,
|
|
3728
|
-
operation_kind: this.stateOperationKind(operation),
|
|
3729
|
-
requested_bytes: operation.prepareWrite?.size,
|
|
3730
|
-
pending_state_operations: this.stateOperationRequests.size
|
|
3731
|
-
});
|
|
3732
|
-
this.publishState();
|
|
3733
|
-
return request.promise;
|
|
3734
|
-
}
|
|
3735
|
-
deliverUpdate(update) {
|
|
3736
|
-
if (update.outputBlob != null) {
|
|
3737
|
-
let id = String(update.outputBlob.blob?.id ?? "");
|
|
3738
|
-
let matchedByOrder = false;
|
|
3739
|
-
if (!id) {
|
|
3740
|
-
id = String(this.state.outputBlobRequests[0]?.id ?? "");
|
|
3741
|
-
matchedByOrder = id !== "";
|
|
3742
|
-
}
|
|
3743
|
-
const request = this.outputBlobRequests.get(id);
|
|
3744
|
-
if (request == null) {
|
|
3745
|
-
this.log("warn", "unmatched output blob update ignored", {
|
|
3746
|
-
blob_id: id || void 0,
|
|
3747
|
-
status_code: update.outputBlob.status?.code,
|
|
3748
|
-
pending_output_blob_requests: this.outputBlobRequests.size
|
|
3749
|
-
});
|
|
3750
|
-
return;
|
|
3751
|
-
}
|
|
3752
|
-
this.outputBlobRequests.delete(id);
|
|
3753
|
-
this.state.outputBlobRequests = this.state.outputBlobRequests.filter((item) => item.id !== id);
|
|
3754
|
-
this.log("debug", "output blob update matched", {
|
|
3755
|
-
blob_id: id,
|
|
3756
|
-
status_code: update.outputBlob.status?.code,
|
|
3757
|
-
chunk_count: update.outputBlob.blob?.chunks?.length ?? 0,
|
|
3758
|
-
matched_by_order: matchedByOrder,
|
|
3759
|
-
pending_output_blob_requests: this.outputBlobRequests.size
|
|
3760
|
-
});
|
|
3761
|
-
request.resolve(update.outputBlob);
|
|
3762
|
-
this.publishState();
|
|
3763
|
-
return;
|
|
3764
|
-
}
|
|
3765
|
-
const result = update.requestStateOperationResult;
|
|
3766
|
-
if (result != null) {
|
|
3767
|
-
if (typeof result.operationId !== "string" || result.operationId.length === 0) {
|
|
3768
|
-
throw this.latchProtocolError(
|
|
3769
|
-
"Allocation received a request state operation result without an operation ID"
|
|
3770
|
-
);
|
|
3771
|
-
}
|
|
3772
|
-
const id = result.operationId;
|
|
3773
|
-
const request = this.stateOperationRequests.get(id);
|
|
3774
|
-
if (request == null) {
|
|
3775
|
-
this.log("warn", "unmatched request state operation result ignored", {
|
|
3776
|
-
operation_id: id,
|
|
3777
|
-
status_code: result.status?.code,
|
|
3778
|
-
pending_state_operations: this.stateOperationRequests.size
|
|
3779
|
-
});
|
|
3780
|
-
return;
|
|
3781
|
-
}
|
|
3782
|
-
const operation = this.state.requestStateOperations.find((item) => item.operationId === id);
|
|
3783
|
-
this.stateOperationRequests.delete(id);
|
|
3784
|
-
this.state.requestStateOperations = this.state.requestStateOperations.filter(
|
|
3785
|
-
(item) => item.operationId !== id
|
|
3786
|
-
);
|
|
3787
|
-
this.log("debug", "request state operation result matched", {
|
|
3788
|
-
operation_id: id,
|
|
3789
|
-
state_key: operation?.stateKey,
|
|
3790
|
-
operation_kind: this.stateOperationKind(operation),
|
|
3791
|
-
status_code: result.status?.code,
|
|
3792
|
-
pending_state_operations: this.stateOperationRequests.size
|
|
3793
|
-
});
|
|
3794
|
-
request.resolve(result);
|
|
3795
|
-
this.publishState();
|
|
3796
|
-
return;
|
|
3797
|
-
}
|
|
3798
|
-
this.log("warn", "allocation update contained no recognized payload");
|
|
3799
|
-
}
|
|
3800
|
-
watchState(stream) {
|
|
3801
|
-
this.stateStreams.add(stream);
|
|
3802
|
-
this.stateStreamFlows.set(stream, { backpressured: false });
|
|
3803
|
-
this.log("info", "allocation state stream connected", {
|
|
3804
|
-
connected_streams: this.stateStreams.size,
|
|
3805
|
-
state_hash: this.state.sha256Hash,
|
|
3806
|
-
pending_output_blob_requests: this.outputBlobRequests.size,
|
|
3807
|
-
pending_state_operations: this.stateOperationRequests.size
|
|
3808
|
-
});
|
|
3809
|
-
try {
|
|
3810
|
-
stream.on("cancelled", () => this.dropStateStream(stream, "cancelled"));
|
|
3811
|
-
stream.on("close", () => this.dropStateStream(stream, "closed"));
|
|
3812
|
-
stream.on("drain", () => this.drainStateStream(stream));
|
|
3813
|
-
this.writeStateSnapshot(stream, this.snapshotState());
|
|
3814
|
-
if (this.finished) stream.end();
|
|
3815
|
-
} catch (error) {
|
|
3816
|
-
this.dropStateStream(stream, "setup_failed", error);
|
|
3817
|
-
throw error;
|
|
3818
|
-
}
|
|
3819
|
-
}
|
|
3820
|
-
updateStateHash() {
|
|
3821
|
-
const content = JSON.stringify({
|
|
3822
|
-
progress: this.state.progress,
|
|
3823
|
-
outputBlobRequests: this.state.outputBlobRequests,
|
|
3824
|
-
requestStateOperations: this.state.requestStateOperations
|
|
3825
|
-
});
|
|
3826
|
-
this.state.sha256Hash = (0, import_node_crypto5.createHash)("sha256").update(content).digest("hex");
|
|
3827
|
-
}
|
|
3828
|
-
snapshotState() {
|
|
3829
|
-
return nativeStructuredClone(this.state);
|
|
3830
|
-
}
|
|
3831
|
-
publishState() {
|
|
3832
|
-
this.updateStateHash();
|
|
3833
|
-
this.log("debug", "allocation state published", {
|
|
3834
|
-
state_hash: this.state.sha256Hash,
|
|
3835
|
-
connected_streams: this.stateStreams.size,
|
|
3836
|
-
pending_output_blob_requests: this.outputBlobRequests.size,
|
|
3837
|
-
pending_state_operations: this.stateOperationRequests.size,
|
|
3838
|
-
progress_current: this.state.progress?.current,
|
|
3839
|
-
progress_total: this.state.progress?.total
|
|
3840
|
-
});
|
|
3841
|
-
for (const stream of [...this.stateStreams]) {
|
|
3842
|
-
this.writeStateSnapshot(stream, this.snapshotState());
|
|
3843
|
-
}
|
|
3844
|
-
}
|
|
3845
|
-
writeStateSnapshot(stream, snapshot) {
|
|
3846
|
-
const flow = this.stateStreamFlows.get(stream);
|
|
3847
|
-
if (flow == null) return;
|
|
3848
|
-
if (flow.backpressured) {
|
|
3849
|
-
flow.pending = snapshot;
|
|
3850
|
-
return;
|
|
3851
|
-
}
|
|
3852
|
-
try {
|
|
3853
|
-
flow.backpressured = !stream.write(snapshot);
|
|
3854
|
-
} catch (error) {
|
|
3855
|
-
this.dropStateStream(stream, "write_failed", error);
|
|
3856
|
-
}
|
|
3857
|
-
}
|
|
3858
|
-
drainStateStream(stream) {
|
|
3859
|
-
const flow = this.stateStreamFlows.get(stream);
|
|
3860
|
-
if (flow == null) return;
|
|
3861
|
-
flow.backpressured = false;
|
|
3862
|
-
const pending = flow.pending;
|
|
3863
|
-
flow.pending = void 0;
|
|
3864
|
-
if (pending != null) this.writeStateSnapshot(stream, pending);
|
|
3865
|
-
}
|
|
3866
|
-
addBatch(events) {
|
|
3867
|
-
if (this.terminalBatchQueued && events.every((event) => event.finishAllocation == null)) {
|
|
3868
|
-
this.log("error", "execution event rejected after terminal batch", {
|
|
3869
|
-
event_count: events.length,
|
|
3870
|
-
event_kinds: events.map((event) => Object.keys(event)[0] ?? "unknown")
|
|
3871
|
-
});
|
|
3872
|
-
return;
|
|
3873
|
-
}
|
|
3874
|
-
this.executionBatches.push(events);
|
|
3875
|
-
this.log("debug", "execution log batch queued", {
|
|
3876
|
-
event_count: events.length,
|
|
3877
|
-
event_kinds: events.map((event) => Object.keys(event)[0] ?? "unknown"),
|
|
3878
|
-
queued_batches: this.executionBatches.length,
|
|
3879
|
-
waiting_consumers: this.executionBatchWaiters.length
|
|
3880
|
-
});
|
|
3881
|
-
for (const waiter of this.executionBatchWaiters.splice(0)) {
|
|
3882
|
-
this.markTerminalBatchObserved(events);
|
|
3883
|
-
waiter(events);
|
|
3884
|
-
}
|
|
3885
|
-
}
|
|
3886
|
-
queueTerminalBatch(finishAllocation) {
|
|
3887
|
-
if (this.terminalBatchQueued) {
|
|
3888
|
-
this.log("error", "duplicate terminal allocation batch rejected", {
|
|
3889
|
-
outcome_code: finishAllocation.outcomeCode,
|
|
3890
|
-
failure_reason: finishAllocation.failureReason
|
|
3891
|
-
});
|
|
3892
|
-
return;
|
|
3893
|
-
}
|
|
3894
|
-
this.terminalBatchQueued = true;
|
|
3895
|
-
this.addBatch([{ finishAllocation }]);
|
|
3896
|
-
}
|
|
3897
|
-
async getExecutionBatch() {
|
|
3898
|
-
if (this.executionBatches.length > 0) {
|
|
3899
|
-
this.log("debug", "returning queued execution log batch", {
|
|
3900
|
-
event_count: this.executionBatches[0].length,
|
|
3901
|
-
queued_batches: this.executionBatches.length
|
|
3902
|
-
});
|
|
3903
|
-
const events = this.executionBatches[0];
|
|
3904
|
-
this.markTerminalBatchObserved(events);
|
|
3905
|
-
return events;
|
|
3906
|
-
}
|
|
3907
|
-
if (this.finished) {
|
|
3908
|
-
this.log("debug", "returning empty execution log batch for finished allocation");
|
|
3909
|
-
return [];
|
|
3910
|
-
}
|
|
3911
|
-
this.log("debug", "execution log batch consumer waiting", {
|
|
3912
|
-
waiting_consumers: this.executionBatchWaiters.length + 1
|
|
3913
|
-
});
|
|
3914
|
-
return new Promise((resolve) => this.executionBatchWaiters.push(resolve));
|
|
3915
|
-
}
|
|
3916
|
-
advanceExecutionBatch() {
|
|
3917
|
-
const advanced = this.executionBatches.shift();
|
|
3918
|
-
this.log("debug", "execution log batch advanced", {
|
|
3919
|
-
advanced_event_count: advanced?.length ?? 0,
|
|
3920
|
-
queued_batches: this.executionBatches.length
|
|
3921
|
-
});
|
|
3922
|
-
}
|
|
3923
|
-
markTerminalBatchObserved(events) {
|
|
3924
|
-
if (this.terminalBatchObserved || !events.some((event) => event.finishAllocation != null)) {
|
|
3925
|
-
return;
|
|
3926
|
-
}
|
|
3927
|
-
this.terminalBatchObserved = true;
|
|
3928
|
-
this.log("debug", "terminal execution log batch observed by consumer");
|
|
3929
|
-
for (const waiter of this.terminalBatchObservedWaiters.splice(0)) waiter();
|
|
3930
|
-
}
|
|
3931
|
-
get isFinished() {
|
|
3932
|
-
return this.finished;
|
|
3933
|
-
}
|
|
3934
|
-
finish() {
|
|
3935
|
-
if (this.finished) {
|
|
3936
|
-
this.log("debug", "allocation finish ignored because runner is already finished");
|
|
3937
|
-
return;
|
|
3938
|
-
}
|
|
3939
|
-
this.finished = true;
|
|
3940
|
-
this.abortExecution(new FunctionError("Allocation finished"));
|
|
3941
|
-
this.log("info", "allocation runner finished", {
|
|
3942
|
-
duration_ms: Date.now() - this.startedAt,
|
|
3943
|
-
queued_execution_batches: this.executionBatches.length,
|
|
3944
|
-
state_streams: this.stateStreams.size,
|
|
3945
|
-
event_log_streams: this.eventReadStreams.size
|
|
3946
|
-
});
|
|
3947
|
-
for (const stream of [...this.stateStreams]) {
|
|
3948
|
-
try {
|
|
3949
|
-
stream.end();
|
|
3950
|
-
} catch (error) {
|
|
3951
|
-
this.dropStateStream(stream, "end_failed", error);
|
|
3952
|
-
}
|
|
3953
|
-
}
|
|
3954
|
-
for (const stream of [...this.eventReadStreams]) {
|
|
3955
|
-
try {
|
|
3956
|
-
stream.end();
|
|
3957
|
-
} catch (error) {
|
|
3958
|
-
this.dropEventReadStream(stream, "end_failed", error);
|
|
3959
|
-
}
|
|
3960
|
-
}
|
|
3961
|
-
for (const waiter of this.executionBatchWaiters.splice(0)) waiter([]);
|
|
3962
|
-
}
|
|
3963
|
-
dropStateStream(stream, reason, error) {
|
|
3964
|
-
if (!this.stateStreams.delete(stream)) return;
|
|
3965
|
-
this.stateStreamFlows.delete(stream);
|
|
3966
|
-
this.log("info", "allocation state stream disconnected", {
|
|
3967
|
-
reason,
|
|
3968
|
-
connected_streams: this.stateStreams.size
|
|
3969
|
-
}, error);
|
|
3970
|
-
}
|
|
3971
|
-
dropEventReadStream(stream, reason, error) {
|
|
3972
|
-
if (!this.eventReadStreams.delete(stream)) return;
|
|
3973
|
-
this.log("info", "event log read stream disconnected", {
|
|
3974
|
-
reason,
|
|
3975
|
-
connected_streams: this.eventReadStreams.size,
|
|
3976
|
-
has_pending_read: this.currentRead != null
|
|
3977
|
-
}, error);
|
|
3978
|
-
}
|
|
3979
|
-
stateOperationKind(operation) {
|
|
3980
|
-
if (operation?.prepareRead != null) return "prepare_read";
|
|
3981
|
-
if (operation?.prepareWrite != null) return "prepare_write";
|
|
3982
|
-
if (operation?.commitWrite != null) return "commit_write";
|
|
3983
|
-
return "unknown";
|
|
3984
|
-
}
|
|
3985
|
-
blobLogFields(blob) {
|
|
3986
|
-
const chunks = blob?.chunks ?? [];
|
|
3987
|
-
return {
|
|
3988
|
-
blob_id: blob?.id,
|
|
3989
|
-
chunk_count: chunks.length,
|
|
3990
|
-
declared_bytes: chunks.reduce((total, chunk) => total + (chunk.size ?? 0), 0)
|
|
3991
|
-
};
|
|
3992
|
-
}
|
|
3993
|
-
async downloadBlob(blob, purpose, fields = {}) {
|
|
3994
|
-
if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
|
|
3995
|
-
const startedAt = Date.now();
|
|
3996
|
-
this.log("debug", "blob download starting", {
|
|
3997
|
-
purpose,
|
|
3998
|
-
...fields,
|
|
3999
|
-
...this.blobLogFields(blob)
|
|
4000
|
-
});
|
|
4001
|
-
try {
|
|
4002
|
-
const data = await downloadBlob(
|
|
4003
|
-
blob,
|
|
4004
|
-
(message, details) => this.log("debug", `blob ${message}`, {
|
|
4005
|
-
purpose,
|
|
4006
|
-
...fields,
|
|
4007
|
-
...details
|
|
4008
|
-
}),
|
|
4009
|
-
this.controller.signal
|
|
4010
|
-
);
|
|
4011
|
-
this.log("debug", "blob download completed", {
|
|
4012
|
-
purpose,
|
|
4013
|
-
...fields,
|
|
4014
|
-
...this.blobLogFields(blob),
|
|
4015
|
-
downloaded_bytes: data.byteLength,
|
|
4016
|
-
duration_ms: Date.now() - startedAt
|
|
4017
|
-
});
|
|
4018
|
-
return data;
|
|
4019
|
-
} catch (error) {
|
|
4020
|
-
this.log("error", "blob download failed", {
|
|
4021
|
-
purpose,
|
|
4022
|
-
...fields,
|
|
4023
|
-
...this.blobLogFields(blob),
|
|
4024
|
-
duration_ms: Date.now() - startedAt
|
|
4025
|
-
}, error);
|
|
4026
|
-
throw error;
|
|
4027
|
-
}
|
|
4028
|
-
}
|
|
4029
|
-
async downloadSerializedObject(object, blob, purpose, fields = {}) {
|
|
4030
|
-
if (object == null) throw new Error(`Missing serialized object for ${purpose}`);
|
|
4031
|
-
if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
|
|
4032
|
-
const startedAt = Date.now();
|
|
4033
|
-
this.log("debug", "serialized object download starting", {
|
|
4034
|
-
purpose,
|
|
4035
|
-
...fields,
|
|
4036
|
-
...this.blobLogFields(blob),
|
|
4037
|
-
object_offset: object.offset ?? 0,
|
|
4038
|
-
object_bytes: object.manifest?.size,
|
|
4039
|
-
metadata_bytes: object.manifest?.metadataSize,
|
|
4040
|
-
content_type: object.manifest?.contentType,
|
|
4041
|
-
encoding: object.manifest?.encoding
|
|
4042
|
-
});
|
|
4043
|
-
try {
|
|
4044
|
-
const value = await downloadSerializedObject(
|
|
4045
|
-
object,
|
|
4046
|
-
blob,
|
|
4047
|
-
(message, details) => this.log(
|
|
4048
|
-
"debug",
|
|
4049
|
-
`blob ${message}`,
|
|
4050
|
-
{ purpose, ...fields, ...details }
|
|
4051
|
-
),
|
|
4052
|
-
this.controller.signal
|
|
4053
|
-
);
|
|
4054
|
-
this.log("debug", "serialized object download completed", {
|
|
4055
|
-
purpose,
|
|
4056
|
-
...fields,
|
|
4057
|
-
...this.blobLogFields(blob),
|
|
4058
|
-
object_bytes: value.data.byteLength + value.metadata.byteLength,
|
|
4059
|
-
data_bytes: value.data.byteLength,
|
|
4060
|
-
metadata_bytes: value.metadata.byteLength,
|
|
4061
|
-
duration_ms: Date.now() - startedAt
|
|
4062
|
-
});
|
|
4063
|
-
return value;
|
|
4064
|
-
} catch (error) {
|
|
4065
|
-
this.log("error", "serialized object download failed", {
|
|
4066
|
-
purpose,
|
|
4067
|
-
...fields,
|
|
4068
|
-
...this.blobLogFields(blob),
|
|
4069
|
-
duration_ms: Date.now() - startedAt
|
|
4070
|
-
}, error);
|
|
4071
|
-
throw error;
|
|
4072
|
-
}
|
|
4073
|
-
}
|
|
4074
|
-
async uploadBlob(blob, data, purpose, fields = {}) {
|
|
4075
|
-
if (blob == null) throw new Error(`Missing BLOB for ${purpose}`);
|
|
4076
|
-
const startedAt = Date.now();
|
|
4077
|
-
this.log("debug", "blob upload starting", {
|
|
4078
|
-
purpose,
|
|
4079
|
-
...fields,
|
|
4080
|
-
...this.blobLogFields(blob),
|
|
4081
|
-
upload_bytes: data.byteLength
|
|
4082
|
-
});
|
|
4083
|
-
try {
|
|
4084
|
-
const uploaded = await uploadBlob(
|
|
4085
|
-
blob,
|
|
4086
|
-
data,
|
|
4087
|
-
(message, details) => this.log("debug", `blob ${message}`, {
|
|
4088
|
-
purpose,
|
|
4089
|
-
...fields,
|
|
4090
|
-
...details
|
|
4091
|
-
}),
|
|
4092
|
-
this.controller.signal
|
|
4093
|
-
);
|
|
4094
|
-
this.log("debug", "blob upload completed", {
|
|
4095
|
-
purpose,
|
|
4096
|
-
...fields,
|
|
4097
|
-
...this.blobLogFields(uploaded),
|
|
4098
|
-
upload_bytes: data.byteLength,
|
|
4099
|
-
duration_ms: Date.now() - startedAt
|
|
4100
|
-
});
|
|
4101
|
-
return uploaded;
|
|
4102
|
-
} catch (error) {
|
|
4103
|
-
this.log("error", "blob upload failed", {
|
|
4104
|
-
purpose,
|
|
4105
|
-
...fields,
|
|
4106
|
-
...this.blobLogFields(blob),
|
|
4107
|
-
upload_bytes: data.byteLength,
|
|
4108
|
-
duration_ms: Date.now() - startedAt
|
|
4109
|
-
}, error);
|
|
4110
|
-
throw error;
|
|
4111
|
-
}
|
|
4112
|
-
}
|
|
4113
|
-
log(level, message, fields = {}, error) {
|
|
4114
|
-
this.runWithoutReplayOwnership(() => {
|
|
4115
|
-
writeStructuredOutput("stderr", () => {
|
|
4116
|
-
const rendered = error instanceof Error ? error.stack?.split("\n") ?? [`${error.name}: ${error.message}`] : error == null ? void 0 : [String(error)];
|
|
4117
|
-
return {
|
|
4118
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4119
|
-
level,
|
|
4120
|
-
component: "typescript_function_executor_allocation",
|
|
4121
|
-
message,
|
|
4122
|
-
allocation_id: this.allocation.allocationId,
|
|
4123
|
-
request_id: this.allocation.requestId,
|
|
4124
|
-
function_call_id: this.allocation.functionCallId,
|
|
4125
|
-
namespace: this.functionRef.namespace,
|
|
4126
|
-
application: this.functionRef.applicationName,
|
|
4127
|
-
application_version: this.functionRef.applicationVersion,
|
|
4128
|
-
function: this.functionRef.functionName,
|
|
4129
|
-
elapsed_ms: Date.now() - this.startedAt,
|
|
4130
|
-
...fields,
|
|
4131
|
-
...rendered == null ? {} : { error: rendered }
|
|
4132
|
-
};
|
|
4133
|
-
});
|
|
4134
|
-
});
|
|
4135
|
-
}
|
|
4136
|
-
runWithoutReplayOwnership(callback) {
|
|
4137
|
-
return this.replay?.runWithoutCausalOwnership(callback) ?? callback();
|
|
4138
|
-
}
|
|
4139
|
-
};
|
|
4140
|
-
|
|
4141
|
-
// src/function-executor/service.ts
|
|
4142
|
-
var SHUTDOWN_TERMINAL_DRAIN_TIMEOUT_MS = 1e3;
|
|
4143
|
-
var nativeImport = (0, import_node_vm.compileFunction)(
|
|
4144
|
-
"return import(specifier)",
|
|
4145
|
-
["specifier"],
|
|
4146
|
-
{ importModuleDynamically: import_node_vm.constants.USE_MAIN_CONTEXT_DEFAULT_LOADER }
|
|
4147
|
-
);
|
|
4148
|
-
function rpcError(code, details) {
|
|
4149
|
-
return Object.assign(new Error(details), { code, details, metadata: new grpc.Metadata() });
|
|
4150
|
-
}
|
|
4151
|
-
function requireSafeSize(value, field) {
|
|
4152
|
-
if (!Number.isSafeInteger(value) || Number(value) < 0) {
|
|
4153
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `${field} must be a non-negative safe integer`);
|
|
4154
|
-
}
|
|
4155
|
-
return Number(value);
|
|
4156
|
-
}
|
|
4157
|
-
function validateSerializedObjectManifest(manifest, field) {
|
|
4158
|
-
if (manifest == null) {
|
|
4159
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest is required`);
|
|
4160
|
-
}
|
|
4161
|
-
if (manifest.encoding == null) {
|
|
4162
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.encoding is required`);
|
|
4163
|
-
}
|
|
4164
|
-
if (manifest.encodingVersion == null) {
|
|
4165
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.encoding_version is required`);
|
|
4166
|
-
}
|
|
4167
|
-
requireSafeSize(manifest.encodingVersion, `${field}.manifest.encoding_version`);
|
|
4168
|
-
requireSafeSize(manifest.size, `${field}.manifest.size`);
|
|
4169
|
-
if (manifest.metadataSize == null) {
|
|
4170
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.metadata_size is required`);
|
|
4171
|
-
}
|
|
4172
|
-
const metadataSize = requireSafeSize(manifest.metadataSize, `${field}.manifest.metadata_size`);
|
|
4173
|
-
if (metadataSize > Number(manifest.size)) {
|
|
4174
|
-
throw rpcError(
|
|
4175
|
-
grpc.status.INVALID_ARGUMENT,
|
|
4176
|
-
`${field}.manifest.metadata_size cannot exceed size`
|
|
4177
|
-
);
|
|
4178
|
-
}
|
|
4179
|
-
if (typeof manifest.sha256Hash !== "string" || manifest.sha256Hash.length === 0) {
|
|
4180
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.manifest.sha256_hash is required`);
|
|
4181
|
-
}
|
|
4182
|
-
}
|
|
4183
|
-
function validateBlob(blob, field) {
|
|
4184
|
-
if (blob == null) throw rpcError(grpc.status.INVALID_ARGUMENT, `${field} is required`);
|
|
4185
|
-
if (!Array.isArray(blob.chunks)) {
|
|
4186
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.chunks must be an array`);
|
|
4187
|
-
}
|
|
4188
|
-
blob.chunks.forEach((chunk, index) => {
|
|
4189
|
-
if (chunk == null || typeof chunk.uri !== "string" || chunk.uri.length === 0) {
|
|
4190
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `${field}.chunks[${index}].uri is required`);
|
|
4191
|
-
}
|
|
4192
|
-
requireSafeSize(chunk.size, `${field}.chunks[${index}].size`);
|
|
4193
|
-
});
|
|
4194
|
-
}
|
|
4195
|
-
function validateNewAllocation(allocation) {
|
|
4196
|
-
if (allocation == null) {
|
|
4197
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, "CreateAllocationRequest.allocation is required");
|
|
4198
|
-
}
|
|
4199
|
-
for (const [field, value] of [
|
|
4200
|
-
["request_id", allocation.requestId],
|
|
4201
|
-
["function_call_id", allocation.functionCallId],
|
|
4202
|
-
["allocation_id", allocation.allocationId]
|
|
4203
|
-
]) {
|
|
4204
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
4205
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `Allocation.${field} is required`);
|
|
4206
|
-
}
|
|
4207
|
-
}
|
|
4208
|
-
const inputs = allocation.inputs;
|
|
4209
|
-
if (inputs == null) {
|
|
4210
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, "Allocation.inputs is required");
|
|
4211
|
-
}
|
|
4212
|
-
if (!Array.isArray(inputs.args) || !Array.isArray(inputs.argBlobs)) {
|
|
4213
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, "Allocation input args and arg_blobs must be arrays");
|
|
4214
|
-
}
|
|
4215
|
-
if (inputs.args.length !== inputs.argBlobs.length) {
|
|
4216
|
-
throw rpcError(
|
|
4217
|
-
grpc.status.INVALID_ARGUMENT,
|
|
4218
|
-
`Mismatched function arguments and argument blobs lengths, ${inputs.args.length} != ${inputs.argBlobs.length}`
|
|
4219
|
-
);
|
|
4220
|
-
}
|
|
4221
|
-
inputs.args.forEach((argument, index) => {
|
|
4222
|
-
if (argument == null) {
|
|
4223
|
-
throw rpcError(grpc.status.INVALID_ARGUMENT, `Allocation.inputs.args[${index}] is required`);
|
|
4224
|
-
}
|
|
4225
|
-
requireSafeSize(argument.offset, `Allocation.inputs.args[${index}].offset`);
|
|
4226
|
-
validateSerializedObjectManifest(argument.manifest, `Allocation.inputs.args[${index}]`);
|
|
4227
|
-
validateBlob(inputs.argBlobs[index], `Allocation.inputs.arg_blobs[${index}]`);
|
|
4228
|
-
});
|
|
4229
|
-
validateBlob(inputs.requestErrorBlob, "Allocation.inputs.request_error_blob");
|
|
4230
|
-
}
|
|
4231
|
-
function safeArchivePath(root, archivePath) {
|
|
4232
|
-
const output = import_node_path.default.resolve(root, archivePath);
|
|
4233
|
-
if (output !== root && !output.startsWith(`${root}${import_node_path.default.sep}`)) {
|
|
4234
|
-
throw new Error(`Application archive contains unsafe path '${archivePath}'`);
|
|
4235
|
-
}
|
|
4236
|
-
return output;
|
|
4237
|
-
}
|
|
4238
|
-
async function unpackApplicationCode(data) {
|
|
4239
|
-
const root = await (0, import_promises2.mkdtemp)(import_node_path.default.join(import_node_os.default.tmpdir(), "tensorlake-typescript-app-"));
|
|
4240
|
-
try {
|
|
4241
|
-
const files = (0, import_fflate.unzipSync)(data);
|
|
4242
|
-
for (const [name, content] of Object.entries(files)) {
|
|
4243
|
-
const destination = safeArchivePath(root, name);
|
|
4244
|
-
await (0, import_promises2.mkdir)(import_node_path.default.dirname(destination), { recursive: true });
|
|
4245
|
-
await (0, import_promises2.writeFile)(destination, content, { mode: 384 });
|
|
4246
|
-
}
|
|
4247
|
-
const manifest = JSON.parse(
|
|
4248
|
-
await (0, import_promises2.readFile)(import_node_path.default.join(root, ".tensorlake_code_manifest.json"), "utf8")
|
|
4249
|
-
);
|
|
4250
|
-
return { root, manifest };
|
|
4251
|
-
} catch (error) {
|
|
4252
|
-
await (0, import_promises2.rm)(root, { recursive: true, force: true });
|
|
4253
|
-
throw error;
|
|
4254
|
-
}
|
|
4255
|
-
}
|
|
4256
|
-
var FunctionExecutorService = class {
|
|
4257
|
-
allocations = /* @__PURE__ */ new Map();
|
|
4258
|
-
functionRef;
|
|
4259
|
-
definition;
|
|
4260
|
-
applicationDefinition;
|
|
4261
|
-
initialization;
|
|
4262
|
-
stopping = false;
|
|
4263
|
-
shutdownPromise;
|
|
4264
|
-
implementation = {
|
|
4265
|
-
initialize: this.initialize.bind(this),
|
|
4266
|
-
listAllocations: this.listAllocations.bind(this),
|
|
4267
|
-
createAllocation: this.createAllocation.bind(this),
|
|
4268
|
-
deleteAllocation: this.deleteAllocation.bind(this),
|
|
4269
|
-
watchAllocationState: this.watchAllocationState.bind(this),
|
|
4270
|
-
sendAllocationUpdate: this.sendAllocationUpdate.bind(this),
|
|
4271
|
-
getAllocationExecutionLogBatch: this.getAllocationExecutionLogBatch.bind(this),
|
|
4272
|
-
advanceAllocationExecutionLogBatch: this.advanceAllocationExecutionLogBatch.bind(this),
|
|
4273
|
-
watchAllocationEventLogReads: this.watchAllocationEventLogReads.bind(this),
|
|
4274
|
-
sendAllocationEventLogReadResponse: this.sendAllocationEventLogReadResponse.bind(this),
|
|
4275
|
-
checkHealth: this.checkHealth.bind(this),
|
|
4276
|
-
getInfo: this.getInfo.bind(this)
|
|
4277
|
-
};
|
|
4278
|
-
shutdown(reason = new Error("Function executor is shutting down")) {
|
|
4279
|
-
if (this.shutdownPromise != null) return this.shutdownPromise;
|
|
4280
|
-
this.stopping = true;
|
|
4281
|
-
const shutdown = (async () => {
|
|
4282
|
-
this.log("info", "function executor service shutdown requested", {
|
|
4283
|
-
allocation_count: this.allocations.size
|
|
4284
|
-
});
|
|
4285
|
-
const allocations = [...this.allocations.values()];
|
|
4286
|
-
for (const allocation of allocations) allocation.cancel(reason);
|
|
4287
|
-
await Promise.all(allocations.map((allocation) => allocation.waitForCompletion()));
|
|
4288
|
-
const terminalObservations = await Promise.all(
|
|
4289
|
-
allocations.map(
|
|
4290
|
-
(allocation) => allocation.waitForTerminalBatchObserved(SHUTDOWN_TERMINAL_DRAIN_TIMEOUT_MS)
|
|
4291
|
-
)
|
|
4292
|
-
);
|
|
4293
|
-
const unobservedCount = terminalObservations.filter((observed) => !observed).length;
|
|
4294
|
-
if (unobservedCount > 0) {
|
|
4295
|
-
this.log(
|
|
4296
|
-
"warn",
|
|
4297
|
-
"shutdown terminal execution batches were not observed before drain deadline",
|
|
4298
|
-
{ unobserved_allocation_count: unobservedCount }
|
|
4299
|
-
);
|
|
4300
|
-
}
|
|
4301
|
-
this.log("info", "function executor service shutdown completed", {
|
|
4302
|
-
allocation_count: allocations.length
|
|
4303
|
-
});
|
|
4304
|
-
})();
|
|
4305
|
-
this.shutdownPromise = shutdown;
|
|
4306
|
-
return shutdown;
|
|
4307
|
-
}
|
|
4308
|
-
initialize(call, callback) {
|
|
4309
|
-
if (this.stopping) {
|
|
4310
|
-
callback(rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down"));
|
|
4311
|
-
return;
|
|
4312
|
-
}
|
|
4313
|
-
const startedAt = Date.now();
|
|
4314
|
-
this.log("info", "initialize RPC received", {
|
|
4315
|
-
...this.functionLogFields(call.request.function),
|
|
4316
|
-
application_code_bytes: call.request.applicationCode?.data?.length
|
|
4317
|
-
});
|
|
4318
|
-
this.emitInitializationEvent(call.request.function, "function_executor_initialization_started", "Initializing function executor");
|
|
4319
|
-
void this.initializeAsync(call.request).then(
|
|
4320
|
-
() => {
|
|
4321
|
-
this.log("info", "initialize RPC completed", {
|
|
4322
|
-
...this.functionLogFields(call.request.function),
|
|
4323
|
-
duration_ms: Date.now() - startedAt,
|
|
4324
|
-
outcome: "success"
|
|
4325
|
-
});
|
|
4326
|
-
this.emitInitializationEvent(call.request.function, "function_executor_initialization_finished", "Function executor initialization completed");
|
|
4327
|
-
callback(null, { outcomeCode: "INITIALIZATION_OUTCOME_CODE_SUCCESS" });
|
|
4328
|
-
},
|
|
4329
|
-
(error) => {
|
|
4330
|
-
const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
4331
|
-
this.log("error", "initialize RPC failed", {
|
|
4332
|
-
...this.functionLogFields(call.request.function),
|
|
4333
|
-
duration_ms: Date.now() - startedAt,
|
|
4334
|
-
outcome: "failure"
|
|
4335
|
-
}, error);
|
|
4336
|
-
this.emitInitializationEvent(
|
|
4337
|
-
call.request.function,
|
|
4338
|
-
"function_executor_initialization_failed",
|
|
4339
|
-
"Function executor initialization failed",
|
|
4340
|
-
error
|
|
4341
|
-
);
|
|
4342
|
-
callback(null, {
|
|
4343
|
-
outcomeCode: "INITIALIZATION_OUTCOME_CODE_FAILURE",
|
|
4344
|
-
failureReason: "INITIALIZATION_FAILURE_REASON_FUNCTION_ERROR",
|
|
4345
|
-
errorMessage: message
|
|
4346
|
-
});
|
|
4347
|
-
}
|
|
4348
|
-
);
|
|
4349
|
-
}
|
|
4350
|
-
emitInitializationEvent(functionRef, event, message, error) {
|
|
4351
|
-
const rendered = error instanceof Error ? error.stack?.split("\n") : error == null ? void 0 : [String(error)];
|
|
4352
|
-
printCloudEvent({
|
|
4353
|
-
...error == null ? {} : { level: "error" },
|
|
4354
|
-
event,
|
|
4355
|
-
message,
|
|
4356
|
-
namespace: functionRef?.namespace,
|
|
4357
|
-
application: functionRef?.applicationName,
|
|
4358
|
-
application_version: functionRef?.applicationVersion,
|
|
4359
|
-
function: functionRef?.functionName,
|
|
4360
|
-
...rendered == null ? {} : { error: rendered }
|
|
4361
|
-
});
|
|
4362
|
-
}
|
|
4363
|
-
async initializeAsync(request) {
|
|
4364
|
-
if (this.definition != null) throw new Error("Function Executor is already initialized");
|
|
4365
|
-
if (this.initialization != null) throw new Error("Function Executor initialization is already in progress");
|
|
4366
|
-
const initialization = this.initializeAttempt(request);
|
|
4367
|
-
this.initialization = initialization;
|
|
4368
|
-
try {
|
|
4369
|
-
await initialization;
|
|
4370
|
-
} finally {
|
|
4371
|
-
if (this.initialization === initialization) this.initialization = void 0;
|
|
4372
|
-
}
|
|
4373
|
-
}
|
|
4374
|
-
async initializeAttempt(request) {
|
|
4375
|
-
if (request.function == null) throw new Error("InitializeRequest.function is required");
|
|
4376
|
-
for (const [field, value] of [
|
|
4377
|
-
["namespace", request.function.namespace],
|
|
4378
|
-
["application_name", request.function.applicationName],
|
|
4379
|
-
["function_name", request.function.functionName],
|
|
4380
|
-
["application_version", request.function.applicationVersion]
|
|
4381
|
-
]) {
|
|
4382
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
4383
|
-
throw new Error(`InitializeRequest.function.${field} is required`);
|
|
4384
|
-
}
|
|
4385
|
-
}
|
|
4386
|
-
if (request.applicationCode == null) throw new Error("InitializeRequest.application_code is required");
|
|
4387
|
-
if (request.applicationCode.manifest == null) throw new Error("InitializeRequest.application_code.manifest is required");
|
|
4388
|
-
if (request.applicationCode.data == null) throw new Error("InitializeRequest.application_code.data is required");
|
|
4389
|
-
const codeManifest = request.applicationCode.manifest;
|
|
4390
|
-
if (codeManifest.encoding == null) throw new Error("InitializeRequest.application_code.manifest.encoding is required");
|
|
4391
|
-
if (codeManifest.encodingVersion == null) throw new Error("InitializeRequest.application_code.manifest.encoding_version is required");
|
|
4392
|
-
if (codeManifest.size == null) throw new Error("InitializeRequest.application_code.manifest.size is required");
|
|
4393
|
-
if (codeManifest.sha256Hash == null) throw new Error("InitializeRequest.application_code.manifest.sha256_hash is required");
|
|
4394
|
-
if (!(/* @__PURE__ */ new Set(["SERIALIZED_OBJECT_ENCODING_BINARY_ZIP", 4])).has(codeManifest.encoding)) {
|
|
4395
|
-
throw new Error(
|
|
4396
|
-
`Invalid application code encoding: ${String(codeManifest.encoding)}. Expected: BINARY_ZIP`
|
|
4397
|
-
);
|
|
4398
|
-
}
|
|
4399
|
-
const declaredSize = requireSafeSize(
|
|
4400
|
-
codeManifest.size,
|
|
4401
|
-
"InitializeRequest.application_code.manifest.size"
|
|
4402
|
-
);
|
|
4403
|
-
const metadataSize = codeManifest.metadataSize == null ? 0 : requireSafeSize(
|
|
4404
|
-
codeManifest.metadataSize,
|
|
4405
|
-
"InitializeRequest.application_code.manifest.metadata_size"
|
|
4406
|
-
);
|
|
4407
|
-
if (metadataSize > declaredSize) {
|
|
4408
|
-
throw new Error(
|
|
4409
|
-
"InitializeRequest.application_code.manifest.metadata_size cannot exceed size"
|
|
4410
|
-
);
|
|
4411
|
-
}
|
|
4412
|
-
const archive = new Uint8Array(request.applicationCode.data);
|
|
4413
|
-
this.log("debug", "unpacking application code", {
|
|
4414
|
-
...this.functionLogFields(request.function),
|
|
4415
|
-
application_code_bytes: archive.byteLength
|
|
4416
|
-
});
|
|
4417
|
-
const unpackStartedAt = Date.now();
|
|
4418
|
-
const { root, manifest } = await unpackApplicationCode(archive);
|
|
4419
|
-
let registrySnapshot;
|
|
4420
|
-
try {
|
|
4421
|
-
this.log("debug", "application code unpacked", {
|
|
4422
|
-
...this.functionLogFields(request.function),
|
|
4423
|
-
duration_ms: Date.now() - unpackStartedAt,
|
|
4424
|
-
manifest_format_version: manifest.format_version,
|
|
4425
|
-
manifest_runtime: manifest.runtime,
|
|
4426
|
-
manifest_function_count: Object.keys(manifest.functions ?? {}).length
|
|
4427
|
-
});
|
|
4428
|
-
if (manifest.format_version !== 2 || manifest.runtime !== "typescript") {
|
|
4429
|
-
throw new Error("Application code is not a Tensorlake TypeScript bundle");
|
|
4430
|
-
}
|
|
4431
|
-
const nodeMajor = Number(process.versions.node.split(".")[0]);
|
|
4432
|
-
if ((manifest.minimum_node_major ?? 24) > nodeMajor) {
|
|
4433
|
-
throw new Error(`Application requires Node ${manifest.minimum_node_major} or newer; executor has ${nodeMajor}`);
|
|
4434
|
-
}
|
|
4435
|
-
const moduleName = manifest.module;
|
|
4436
|
-
if (!moduleName) throw new Error("Application code manifest does not name a runtime module");
|
|
4437
|
-
const functionName = String(request.function.functionName);
|
|
4438
|
-
if (manifest.functions?.[functionName] == null) {
|
|
4439
|
-
throw new Error(`Function '${functionName}' is not present in the application bundle`);
|
|
4440
|
-
}
|
|
4441
|
-
const modulePath = safeArchivePath(root, moduleName);
|
|
4442
|
-
this.log("debug", "importing application runtime module", {
|
|
4443
|
-
...this.functionLogFields(request.function),
|
|
4444
|
-
module: moduleName
|
|
4445
|
-
});
|
|
4446
|
-
const importStartedAt = Date.now();
|
|
4447
|
-
registrySnapshot = snapshotRegistry();
|
|
4448
|
-
const runtime = await nativeImport(`${(0, import_node_url2.pathToFileURL)(modulePath).href}?executor=${Date.now()}`);
|
|
4449
|
-
if (typeof runtime.__tensorlakeGetFunction !== "function") {
|
|
4450
|
-
throw new Error("Application runtime does not export __tensorlakeGetFunction");
|
|
4451
|
-
}
|
|
4452
|
-
const definition = runtime.__tensorlakeGetFunction(functionName);
|
|
4453
|
-
if (definition?.name !== functionName) {
|
|
4454
|
-
throw new Error(
|
|
4455
|
-
`Application runtime resolved function '${functionName}' as '${String(definition?.name)}'`
|
|
4456
|
-
);
|
|
4457
|
-
}
|
|
4458
|
-
const applicationName = String(request.function.applicationName);
|
|
4459
|
-
const applicationDefinition = runtime.__tensorlakeGetFunction(applicationName);
|
|
4460
|
-
if (applicationDefinition?.name !== applicationName) {
|
|
4461
|
-
throw new Error(
|
|
4462
|
-
`Application runtime resolved application '${applicationName}' as '${String(applicationDefinition?.name)}'`
|
|
4463
|
-
);
|
|
4464
|
-
}
|
|
4465
|
-
if (applicationDefinition.application == null) {
|
|
4466
|
-
throw new Error(`'${applicationName}' is not a Tensorlake application`);
|
|
4467
|
-
}
|
|
4468
|
-
this.functionRef = request.function;
|
|
4469
|
-
this.definition = definition;
|
|
4470
|
-
this.applicationDefinition = applicationDefinition;
|
|
4471
|
-
this.log("debug", "application function resolved", {
|
|
4472
|
-
...this.functionLogFields(request.function),
|
|
4473
|
-
duration_ms: Date.now() - importStartedAt,
|
|
4474
|
-
parameter_count: definition.parameters.length,
|
|
4475
|
-
application_max_retries: applicationDefinition.application.retries.maxRetries
|
|
4476
|
-
});
|
|
4477
|
-
registrySnapshot = void 0;
|
|
4478
|
-
} catch (error) {
|
|
4479
|
-
if (registrySnapshot != null) restoreRegistry(registrySnapshot);
|
|
4480
|
-
this.functionRef = void 0;
|
|
4481
|
-
this.definition = void 0;
|
|
4482
|
-
this.applicationDefinition = void 0;
|
|
4483
|
-
await (0, import_promises2.rm)(root, { recursive: true, force: true });
|
|
4484
|
-
throw error;
|
|
4485
|
-
}
|
|
4486
|
-
}
|
|
4487
|
-
getAllocation(id) {
|
|
4488
|
-
const allocation = this.allocations.get(id);
|
|
4489
|
-
if (allocation == null) throw rpcError(grpc.status.NOT_FOUND, `Allocation ${id} not found`);
|
|
4490
|
-
return allocation;
|
|
4491
|
-
}
|
|
4492
|
-
listAllocations(_call, callback) {
|
|
4493
|
-
this.log("debug", "list allocations RPC received", { allocation_count: this.allocations.size });
|
|
4494
|
-
callback(null, { allocations: [...this.allocations.values()].map((runner) => runner.allocation) });
|
|
4495
|
-
}
|
|
4496
|
-
createAllocation(call, callback) {
|
|
4497
|
-
void this.createAllocationAsync(call).then(
|
|
4498
|
-
() => callback(null, {}),
|
|
4499
|
-
(error) => {
|
|
4500
|
-
this.log(
|
|
4501
|
-
"error",
|
|
4502
|
-
"create allocation RPC failed",
|
|
4503
|
-
this.allocationLogFields(call.request.allocation),
|
|
4504
|
-
error
|
|
4505
|
-
);
|
|
4506
|
-
callback(error);
|
|
4507
|
-
}
|
|
4508
|
-
);
|
|
4509
|
-
}
|
|
4510
|
-
async createAllocationAsync(call) {
|
|
4511
|
-
if (this.stopping) {
|
|
4512
|
-
throw rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down");
|
|
4513
|
-
}
|
|
4514
|
-
const initialization = this.initialization;
|
|
4515
|
-
if (initialization != null) {
|
|
4516
|
-
this.log("info", "create allocation RPC waiting for initialization", {
|
|
4517
|
-
...this.allocationLogFields(call.request.allocation)
|
|
4518
|
-
});
|
|
4519
|
-
await this.waitForInitialization(call, initialization);
|
|
4520
|
-
this.log("info", "create allocation RPC resumed after initialization", {
|
|
4521
|
-
...this.allocationLogFields(call.request.allocation)
|
|
4522
|
-
});
|
|
4523
|
-
}
|
|
4524
|
-
this.assertAllocationAdmissionActive(call);
|
|
4525
|
-
if (this.stopping) {
|
|
4526
|
-
throw rpcError(grpc.status.UNAVAILABLE, "Function Executor is shutting down");
|
|
4527
|
-
}
|
|
4528
|
-
if (this.definition == null || this.applicationDefinition == null || this.functionRef == null) {
|
|
4529
|
-
throw rpcError(grpc.status.FAILED_PRECONDITION, "Function Executor is not initialized");
|
|
4530
|
-
}
|
|
4531
|
-
const allocation = call.request.allocation;
|
|
4532
|
-
validateNewAllocation(allocation);
|
|
4533
|
-
const id = String(allocation.allocationId);
|
|
4534
|
-
this.log("info", "create allocation RPC received", {
|
|
4535
|
-
...this.allocationLogFields(allocation),
|
|
4536
|
-
replay_mode: allocation?.replayMode,
|
|
4537
|
-
input_count: allocation?.inputs?.args?.length ?? 0
|
|
4538
|
-
});
|
|
4539
|
-
if (this.allocations.has(id)) throw rpcError(grpc.status.ALREADY_EXISTS, `Allocation ${id} already exists`);
|
|
4540
|
-
const runner = new AllocationRunner(
|
|
4541
|
-
allocation,
|
|
4542
|
-
this.functionRef,
|
|
4543
|
-
this.definition,
|
|
4544
|
-
this.applicationDefinition
|
|
4545
|
-
);
|
|
4546
|
-
this.allocations.set(id, runner);
|
|
4547
|
-
runner.start();
|
|
4548
|
-
this.log("info", "allocation runner started", this.allocationLogFields(allocation));
|
|
4549
|
-
}
|
|
4550
|
-
waitForInitialization(call, initialization) {
|
|
4551
|
-
return new Promise((resolve, reject) => {
|
|
4552
|
-
let timer;
|
|
4553
|
-
let settled = false;
|
|
4554
|
-
const deadline = typeof call.getDeadline === "function" ? call.getDeadline() : Infinity;
|
|
4555
|
-
const deadlineMs = deadline instanceof Date ? deadline.getTime() : Number(deadline);
|
|
4556
|
-
const cancellationError = () => Number.isFinite(deadlineMs) && deadlineMs <= Date.now() ? rpcError(
|
|
4557
|
-
grpc.status.DEADLINE_EXCEEDED,
|
|
4558
|
-
"Initialization did not finish before the allocation deadline"
|
|
4559
|
-
) : rpcError(
|
|
4560
|
-
grpc.status.CANCELLED,
|
|
4561
|
-
"Create allocation RPC was cancelled during initialization"
|
|
4562
|
-
);
|
|
4563
|
-
const onCancelled = () => finish(cancellationError());
|
|
4564
|
-
const finish = (error) => {
|
|
4565
|
-
if (settled) return;
|
|
4566
|
-
settled = true;
|
|
4567
|
-
if (timer != null) clearTimeout(timer);
|
|
4568
|
-
if (typeof call.off === "function") call.off("cancelled", onCancelled);
|
|
4569
|
-
if (error == null) resolve();
|
|
4570
|
-
else reject(error);
|
|
4571
|
-
};
|
|
4572
|
-
if (call.cancelled || Number.isFinite(deadlineMs) && deadlineMs <= Date.now()) {
|
|
4573
|
-
finish(cancellationError());
|
|
4574
|
-
return;
|
|
4575
|
-
}
|
|
4576
|
-
if (typeof call.once === "function") call.once("cancelled", onCancelled);
|
|
4577
|
-
if (Number.isFinite(deadlineMs)) {
|
|
4578
|
-
const remainingMs = deadlineMs - Date.now();
|
|
4579
|
-
if (remainingMs <= 0) {
|
|
4580
|
-
finish(rpcError(grpc.status.DEADLINE_EXCEEDED, "Initialization did not finish before the allocation deadline"));
|
|
4581
|
-
return;
|
|
4582
|
-
}
|
|
4583
|
-
timer = setTimeout(() => {
|
|
4584
|
-
finish(rpcError(
|
|
4585
|
-
grpc.status.DEADLINE_EXCEEDED,
|
|
4586
|
-
"Initialization did not finish before the allocation deadline"
|
|
4587
|
-
));
|
|
4588
|
-
}, remainingMs);
|
|
4589
|
-
}
|
|
4590
|
-
initialization.then(
|
|
4591
|
-
() => finish(
|
|
4592
|
-
call.cancelled || Number.isFinite(deadlineMs) && deadlineMs <= Date.now() ? cancellationError() : void 0
|
|
4593
|
-
),
|
|
4594
|
-
() => finish(rpcError(
|
|
4595
|
-
grpc.status.FAILED_PRECONDITION,
|
|
4596
|
-
"Function Executor initialization failed"
|
|
4597
|
-
))
|
|
4598
|
-
);
|
|
4599
|
-
});
|
|
4600
|
-
}
|
|
4601
|
-
assertAllocationAdmissionActive(call) {
|
|
4602
|
-
const deadline = typeof call.getDeadline === "function" ? call.getDeadline() : Infinity;
|
|
4603
|
-
const deadlineMs = deadline instanceof Date ? deadline.getTime() : Number(deadline);
|
|
4604
|
-
if (Number.isFinite(deadlineMs) && deadlineMs <= Date.now()) {
|
|
4605
|
-
throw rpcError(grpc.status.DEADLINE_EXCEEDED, "Create allocation RPC deadline exceeded");
|
|
4606
|
-
}
|
|
4607
|
-
if (call.cancelled) {
|
|
4608
|
-
throw rpcError(grpc.status.CANCELLED, "Create allocation RPC was cancelled before allocation start");
|
|
4609
|
-
}
|
|
4610
|
-
}
|
|
4611
|
-
deleteAllocation(call, callback) {
|
|
4612
|
-
try {
|
|
4613
|
-
const id = String(call.request.allocationId ?? "");
|
|
4614
|
-
const allocation = this.getAllocation(id);
|
|
4615
|
-
this.log("debug", "delete allocation RPC received", {
|
|
4616
|
-
allocation_id: id,
|
|
4617
|
-
finished: allocation.isFinished
|
|
4618
|
-
});
|
|
4619
|
-
if (!allocation.isFinished) {
|
|
4620
|
-
throw rpcError(grpc.status.FAILED_PRECONDITION, `Allocation ${id} is still running`);
|
|
4621
|
-
}
|
|
4622
|
-
this.allocations.delete(id);
|
|
4623
|
-
this.log("info", "allocation deleted", { allocation_id: id });
|
|
4624
|
-
callback(null, {});
|
|
4625
|
-
} catch (error) {
|
|
4626
|
-
this.log("error", "delete allocation RPC failed", { allocation_id: call.request.allocationId }, error);
|
|
4627
|
-
callback(error);
|
|
4628
|
-
}
|
|
4629
|
-
}
|
|
4630
|
-
watchAllocationState(call) {
|
|
4631
|
-
try {
|
|
4632
|
-
const allocationId = String(call.request.allocationId ?? "");
|
|
4633
|
-
this.log("debug", "allocation state stream opening", { allocation_id: allocationId });
|
|
4634
|
-
this.getAllocation(allocationId).watchState(call);
|
|
4635
|
-
} catch (error) {
|
|
4636
|
-
this.log("error", "allocation state stream failed to open", { allocation_id: call.request.allocationId }, error);
|
|
4637
|
-
call.destroy(error);
|
|
4638
|
-
}
|
|
4639
|
-
}
|
|
4640
|
-
sendAllocationUpdate(call, callback) {
|
|
4641
|
-
try {
|
|
4642
|
-
const allocationId = String(call.request.allocationId ?? "");
|
|
4643
|
-
this.log("debug", "allocation update RPC received", {
|
|
4644
|
-
allocation_id: allocationId,
|
|
4645
|
-
update_kind: call.request.outputBlob != null ? "output_blob" : call.request.requestStateOperationResult != null ? "request_state_operation_result" : "unknown"
|
|
4646
|
-
});
|
|
4647
|
-
if (call.request.outputBlob == null && call.request.requestStateOperationResult == null) {
|
|
4648
|
-
throw rpcError(
|
|
4649
|
-
grpc.status.INVALID_ARGUMENT,
|
|
4650
|
-
`Unknown update type in AllocationUpdate for allocation ${allocationId}`
|
|
4651
|
-
);
|
|
4652
|
-
}
|
|
4653
|
-
const allocation = this.getAllocation(allocationId);
|
|
4654
|
-
if (allocation.isFinished) throw rpcError(grpc.status.FAILED_PRECONDITION, "Allocation is already finished");
|
|
4655
|
-
allocation.deliverUpdate(call.request);
|
|
4656
|
-
callback(null, {});
|
|
4657
|
-
} catch (error) {
|
|
4658
|
-
this.log("error", "allocation update RPC failed", { allocation_id: call.request.allocationId }, error);
|
|
4659
|
-
callback(error);
|
|
4660
|
-
}
|
|
4661
|
-
}
|
|
4662
|
-
getAllocationExecutionLogBatch(call, callback) {
|
|
4663
|
-
try {
|
|
4664
|
-
const allocationId = String(call.request.allocationId ?? "");
|
|
4665
|
-
this.log("debug", "execution log batch RPC waiting", { allocation_id: allocationId });
|
|
4666
|
-
const allocation = this.getAllocation(allocationId);
|
|
4667
|
-
void allocation.getExecutionBatch().then(
|
|
4668
|
-
(events) => {
|
|
4669
|
-
this.log("debug", "execution log batch RPC completed", {
|
|
4670
|
-
allocation_id: allocationId,
|
|
4671
|
-
event_count: events.length
|
|
4672
|
-
});
|
|
4673
|
-
callback(null, { events });
|
|
4674
|
-
},
|
|
4675
|
-
(error) => {
|
|
4676
|
-
this.log("error", "execution log batch RPC failed", { allocation_id: allocationId }, error);
|
|
4677
|
-
callback(error);
|
|
4678
|
-
}
|
|
4679
|
-
);
|
|
4680
|
-
} catch (error) {
|
|
4681
|
-
this.log("error", "execution log batch RPC failed", { allocation_id: call.request.allocationId }, error);
|
|
4682
|
-
callback(error);
|
|
4683
|
-
}
|
|
4684
|
-
}
|
|
4685
|
-
advanceAllocationExecutionLogBatch(call, callback) {
|
|
4686
|
-
try {
|
|
4687
|
-
const allocationId = String(call.request.allocationId ?? "");
|
|
4688
|
-
this.log("debug", "advance execution log batch RPC received", { allocation_id: allocationId });
|
|
4689
|
-
this.getAllocation(allocationId).advanceExecutionBatch();
|
|
4690
|
-
callback(null, {});
|
|
4691
|
-
} catch (error) {
|
|
4692
|
-
this.log("error", "advance execution log batch RPC failed", { allocation_id: call.request.allocationId }, error);
|
|
4693
|
-
callback(error);
|
|
4694
|
-
}
|
|
4695
|
-
}
|
|
4696
|
-
watchAllocationEventLogReads(call) {
|
|
4697
|
-
try {
|
|
4698
|
-
const allocationId = String(call.request.allocationId ?? "");
|
|
4699
|
-
const allocation = this.getAllocation(allocationId);
|
|
4700
|
-
this.log("debug", "event log read stream opening", { allocation_id: allocationId });
|
|
4701
|
-
call.sendMetadata(new grpc.Metadata());
|
|
4702
|
-
this.log("debug", "event log read stream established", { allocation_id: allocationId });
|
|
4703
|
-
allocation.watchEventLogReads(call);
|
|
4704
|
-
} catch (error) {
|
|
4705
|
-
this.log("error", "event log read stream failed to open", { allocation_id: call.request.allocationId }, error);
|
|
4706
|
-
call.destroy(error);
|
|
4707
|
-
}
|
|
4708
|
-
}
|
|
4709
|
-
sendAllocationEventLogReadResponse(call, callback) {
|
|
4710
|
-
try {
|
|
4711
|
-
const allocationId = String(call.request.allocationId ?? "");
|
|
4712
|
-
this.log("debug", "event log read response RPC received", {
|
|
4713
|
-
allocation_id: allocationId,
|
|
4714
|
-
entry_count: call.request.entries?.length ?? 0,
|
|
4715
|
-
last_clock: call.request.lastClock,
|
|
4716
|
-
has_more: call.request.hasMore
|
|
4717
|
-
});
|
|
4718
|
-
this.getAllocation(allocationId).deliverEventLogResponse(call.request);
|
|
4719
|
-
callback(null, {});
|
|
4720
|
-
} catch (error) {
|
|
4721
|
-
this.log("error", "event log read response RPC failed", { allocation_id: call.request.allocationId }, error);
|
|
4722
|
-
callback(error);
|
|
4723
|
-
}
|
|
4724
|
-
}
|
|
4725
|
-
checkHealth(_call, callback) {
|
|
4726
|
-
this.log("debug", "health check RPC received", { initialized: this.definition != null });
|
|
4727
|
-
if (this.definition == null) {
|
|
4728
|
-
callback(rpcError(grpc.status.UNAVAILABLE, "Function Executor is not initialized"));
|
|
4729
|
-
} else {
|
|
4730
|
-
callback(null, { healthy: true, statusMessage: "ok" });
|
|
4731
|
-
}
|
|
4732
|
-
}
|
|
4733
|
-
getInfo(_call, callback) {
|
|
4734
|
-
this.log("debug", "executor info RPC received", { protocol_version: "0.1.3" });
|
|
4735
|
-
callback(null, {
|
|
4736
|
-
version: "0.1.3",
|
|
4737
|
-
sdkVersion: SDK_VERSION,
|
|
4738
|
-
sdkLanguage: "typescript",
|
|
4739
|
-
sdkLanguageVersion: process.versions.node
|
|
4740
|
-
});
|
|
4741
|
-
}
|
|
4742
|
-
functionLogFields(functionRef) {
|
|
4743
|
-
return {
|
|
4744
|
-
namespace: functionRef?.namespace,
|
|
4745
|
-
application: functionRef?.applicationName,
|
|
4746
|
-
application_version: functionRef?.applicationVersion,
|
|
4747
|
-
function: functionRef?.functionName
|
|
4748
|
-
};
|
|
4749
|
-
}
|
|
4750
|
-
allocationLogFields(allocation) {
|
|
4751
|
-
return {
|
|
4752
|
-
allocation_id: allocation?.allocationId,
|
|
4753
|
-
request_id: allocation?.requestId,
|
|
4754
|
-
function_call_id: allocation?.functionCallId
|
|
4755
|
-
};
|
|
4756
|
-
}
|
|
4757
|
-
log(level, message, fields = {}, error) {
|
|
4758
|
-
writeStructuredOutput("stderr", () => {
|
|
4759
|
-
const rendered = error instanceof Error ? error.stack?.split("\n") ?? [`${error.name}: ${error.message}`] : error == null ? void 0 : [String(error)];
|
|
4760
|
-
return {
|
|
4761
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4762
|
-
level,
|
|
4763
|
-
component: "typescript_function_executor_service",
|
|
4764
|
-
message,
|
|
4765
|
-
...fields,
|
|
4766
|
-
...rendered == null ? {} : { error: rendered }
|
|
4767
|
-
};
|
|
4768
|
-
});
|
|
4769
|
-
}
|
|
4770
|
-
};
|
|
4771
|
-
|
|
4772
|
-
// src/function-executor/main.ts
|
|
4773
|
-
var SHUTDOWN_GRACE_PERIOD_MS = 1e4;
|
|
4774
|
-
var IPV4_LOOPBACK_ADDRESS = "127.0.0.1";
|
|
4775
|
-
function functionExecutorBindAddress(address) {
|
|
4776
|
-
if (address === "localhost") return IPV4_LOOPBACK_ADDRESS;
|
|
4777
|
-
if (address.startsWith("localhost:")) {
|
|
4778
|
-
return `${IPV4_LOOPBACK_ADDRESS}${address.slice("localhost".length)}`;
|
|
4779
|
-
}
|
|
4780
|
-
return address;
|
|
4781
|
-
}
|
|
4782
|
-
function parseArguments(argv) {
|
|
4783
|
-
const result = { functionExecutorId: "" };
|
|
4784
|
-
for (let index = 0; index < argv.length; index += 1) {
|
|
4785
|
-
const argument = argv[index];
|
|
4786
|
-
const [key, inline] = argument.split("=", 2);
|
|
4787
|
-
const value = inline ?? argv[index + 1];
|
|
4788
|
-
if (key === "--address") result.address = value;
|
|
4789
|
-
else if (key === "--executor-id") result.executorId = value;
|
|
4790
|
-
else if (key === "--function-executor-id") result.functionExecutorId = value ?? "";
|
|
4791
|
-
else continue;
|
|
4792
|
-
if (inline == null) index += 1;
|
|
4793
|
-
}
|
|
4794
|
-
return result;
|
|
4795
|
-
}
|
|
4796
|
-
function serviceDefinition() {
|
|
4797
|
-
const protoRoot = functionExecutorProtoRoot();
|
|
4798
|
-
const protoPath = import_node_path2.default.join(protoRoot, "tensorlake/function_executor/proto/function_executor.proto");
|
|
4799
|
-
const packageDefinition = (0, import_proto_loader.loadSync)(protoPath, {
|
|
4800
|
-
includeDirs: [import_node_path2.default.dirname(protoPath), protoRoot, (0, import_google_proto_files.getProtoPath)()],
|
|
4801
|
-
keepCase: false,
|
|
4802
|
-
longs: Number,
|
|
4803
|
-
enums: String,
|
|
4804
|
-
defaults: false,
|
|
4805
|
-
oneofs: true
|
|
4806
|
-
});
|
|
4807
|
-
const loaded = grpc2.loadPackageDefinition(packageDefinition);
|
|
4808
|
-
return loaded.function_executor_service.FunctionExecutor.service;
|
|
4809
|
-
}
|
|
4810
|
-
function functionExecutorProtoRoot() {
|
|
4811
|
-
const relativeRoots = [
|
|
4812
|
-
// Standalone function-executor capsule.
|
|
4813
|
-
"../../proto/",
|
|
4814
|
-
// Function executor launched directly from the published SDK package.
|
|
4815
|
-
"../../runtime/function-executor/package/proto/",
|
|
4816
|
-
// Development checkout: both language runtimes use the repository source.
|
|
4817
|
-
"../../../proto/"
|
|
4818
|
-
];
|
|
4819
|
-
for (const relativeRoot of relativeRoots) {
|
|
4820
|
-
const protoRoot = (0, import_node_url3.fileURLToPath)(new URL(relativeRoot, importMetaUrl));
|
|
4821
|
-
const protoPath = import_node_path2.default.join(protoRoot, "tensorlake/function_executor/proto/function_executor.proto");
|
|
4822
|
-
if ((0, import_node_fs.existsSync)(protoPath)) return protoRoot;
|
|
4823
|
-
}
|
|
4824
|
-
throw new Error("Cannot find the shared Tensorlake function-executor protobuf definition");
|
|
4825
|
-
}
|
|
4826
|
-
async function main(argv = process.argv.slice(2)) {
|
|
4827
|
-
configureNativeWorker(() => new URL("../native-worker.cjs", importMetaUrl));
|
|
4828
|
-
const nodeMajor = Number(process.versions.node.split(".")[0]);
|
|
4829
|
-
if (nodeMajor < 24) throw new Error(`Tensorlake TypeScript functions require Node 24 or newer; got ${process.versions.node}`);
|
|
4830
|
-
const args = parseArguments(argv);
|
|
4831
|
-
if (!args.address) throw new Error("--address argument is required");
|
|
4832
|
-
if (!args.executorId) throw new Error("--executor-id argument is required");
|
|
4833
|
-
const address = functionExecutorBindAddress(args.address);
|
|
4834
|
-
writeStructuredOutput("stderr", () => ({
|
|
4835
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4836
|
-
level: "info",
|
|
4837
|
-
component: "typescript_function_executor_main",
|
|
4838
|
-
message: "starting TypeScript function executor",
|
|
4839
|
-
address,
|
|
4840
|
-
executor_id: args.executorId,
|
|
4841
|
-
fn_executor_id: args.functionExecutorId,
|
|
4842
|
-
node_version: process.versions.node,
|
|
4843
|
-
pid: process.pid
|
|
4844
|
-
}));
|
|
4845
|
-
const server = new grpc2.Server({
|
|
4846
|
-
"grpc.max_receive_message_length": -1,
|
|
4847
|
-
"grpc.max_send_message_length": -1,
|
|
4848
|
-
"grpc.so_reuseport": 0
|
|
4849
|
-
});
|
|
4850
|
-
const service = new FunctionExecutorService();
|
|
4851
|
-
server.addService(serviceDefinition(), service.implementation);
|
|
4852
|
-
await new Promise((resolve, reject) => {
|
|
4853
|
-
server.bindAsync(address, grpc2.ServerCredentials.createInsecure(), (error) => {
|
|
4854
|
-
if (error != null) reject(error);
|
|
4855
|
-
else resolve();
|
|
4856
|
-
});
|
|
4857
|
-
});
|
|
4858
|
-
writeStructuredOutput("stderr", () => ({
|
|
4859
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4860
|
-
level: "info",
|
|
4861
|
-
component: "typescript_function_executor_main",
|
|
4862
|
-
message: "started TypeScript function executor",
|
|
4863
|
-
address,
|
|
4864
|
-
executor_id: args.executorId,
|
|
4865
|
-
fn_executor_id: args.functionExecutorId,
|
|
4866
|
-
node_version: process.versions.node
|
|
4867
|
-
}));
|
|
4868
|
-
let stopping = false;
|
|
4869
|
-
const shutdown = (signal) => {
|
|
4870
|
-
if (stopping) return;
|
|
4871
|
-
stopping = true;
|
|
4872
|
-
writeStructuredOutput("stderr", () => ({
|
|
4873
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4874
|
-
level: "info",
|
|
4875
|
-
component: "typescript_function_executor_main",
|
|
4876
|
-
message: "stopping TypeScript function executor",
|
|
4877
|
-
signal,
|
|
4878
|
-
executor_id: args.executorId,
|
|
4879
|
-
fn_executor_id: args.functionExecutorId,
|
|
4880
|
-
pid: process.pid
|
|
4881
|
-
}));
|
|
4882
|
-
const forceShutdownTimer = setTimeout(() => {
|
|
4883
|
-
writeStructuredOutput("stderr", () => ({
|
|
4884
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4885
|
-
level: "error",
|
|
4886
|
-
component: "typescript_function_executor_main",
|
|
4887
|
-
message: "forcing TypeScript function executor shutdown after grace period",
|
|
4888
|
-
signal,
|
|
4889
|
-
executor_id: args.executorId,
|
|
4890
|
-
fn_executor_id: args.functionExecutorId,
|
|
4891
|
-
pid: process.pid,
|
|
4892
|
-
grace_period_ms: SHUTDOWN_GRACE_PERIOD_MS
|
|
4893
|
-
}));
|
|
4894
|
-
server.forceShutdown();
|
|
4895
|
-
process.exit(1);
|
|
4896
|
-
}, SHUTDOWN_GRACE_PERIOD_MS);
|
|
4897
|
-
const gracefulShutdown = service.shutdown(new Error(`Function executor received ${signal}`)).then(() => new Promise((resolve, reject) => {
|
|
4898
|
-
server.tryShutdown((error) => {
|
|
4899
|
-
if (error != null) reject(error);
|
|
4900
|
-
else resolve();
|
|
4901
|
-
});
|
|
4902
|
-
}));
|
|
4903
|
-
void gracefulShutdown.then(
|
|
4904
|
-
() => {
|
|
4905
|
-
clearTimeout(forceShutdownTimer);
|
|
4906
|
-
writeStructuredOutput("stderr", () => ({
|
|
4907
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4908
|
-
level: "info",
|
|
4909
|
-
component: "typescript_function_executor_main",
|
|
4910
|
-
message: "stopped TypeScript function executor",
|
|
4911
|
-
signal,
|
|
4912
|
-
executor_id: args.executorId,
|
|
4913
|
-
fn_executor_id: args.functionExecutorId,
|
|
4914
|
-
pid: process.pid
|
|
4915
|
-
}));
|
|
4916
|
-
process.exit(0);
|
|
4917
|
-
},
|
|
4918
|
-
(error) => {
|
|
4919
|
-
clearTimeout(forceShutdownTimer);
|
|
4920
|
-
writeStructuredOutput("stderr", () => ({
|
|
4921
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4922
|
-
level: "error",
|
|
4923
|
-
component: "typescript_function_executor_main",
|
|
4924
|
-
message: "TypeScript function executor shutdown failed",
|
|
4925
|
-
signal,
|
|
4926
|
-
executor_id: args.executorId,
|
|
4927
|
-
fn_executor_id: args.functionExecutorId,
|
|
4928
|
-
pid: process.pid,
|
|
4929
|
-
error: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
|
|
4930
|
-
}));
|
|
4931
|
-
server.forceShutdown();
|
|
4932
|
-
process.exit(1);
|
|
4933
|
-
}
|
|
4934
|
-
);
|
|
4935
|
-
};
|
|
4936
|
-
process.once("SIGINT", () => shutdown("SIGINT"));
|
|
4937
|
-
process.once("SIGTERM", () => shutdown("SIGTERM"));
|
|
4938
|
-
}
|
|
4939
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
4940
|
-
0 && (module.exports = {
|
|
4941
|
-
functionExecutorBindAddress,
|
|
4942
|
-
main
|
|
4943
|
-
});
|
|
4944
|
-
//# sourceMappingURL=main.cjs.map
|