withruntime 0.7.2 → 0.8.0

package/dist/blaxel/process.js

@@ -0,0 +1,526 @@
+import { checkEnvNames, httpDate, parseProcessLine, processLine, randomName, RUNTIME_HOME, toBlaxelPath, toRuntimePath, } from "./context.js";
+import { codeOf, guard, ResponseError, responseError, translate } from "./errors.js";
+/** Commands without an end of their own run as long as Runtime allows. */
+const LONGEST_MS = 86_400_000;
+/** A keepAlive process is killed after this long unless `timeout` says. */
+const KEEP_ALIVE_SECONDS = 600;
+const SIGKILL = 9;
+/** Per sandbox, shared by every object this module makes for it. A fresh
+ * client starts empty and finds processes by their marker. */
+const registries = new Map();
+function registryOf(sandboxId) {
+    let registry = registries.get(sandboxId);
+    if (!registry) {
+        registry = { byName: new Map(), meta: new Map(), processes: new Map() };
+        registries.set(sandboxId, registry);
+    }
+    return registry;
+}
+/** The loopback addresses in /proc/net/tcp and tcp6: a server bound only to
+ * one cannot be reached from outside, so Blaxel does not count it. */
+const LOOPBACK = [
+    "0100007F",
+    "00000000000000000000000001000000",
+    "0000000000000000FFFF00000100007F",
+];
+/** A command that waits until every port listens on an address other than
+ * loopback, up to `seconds`, reading whichever of the kernel's socket tables
+ * exist (a guest without IPv6 has no tcp6). Exits 0 when they all listen. */
+export function portWaitScript(ports, seconds, tables = ["/proc/net/tcp", "/proc/net/tcp6"]) {
+    const hex = ports.map((port) => port.toString(16).toUpperCase().padStart(4, "0"));
+    const skip = LOOPBACK.map((address) => `a[1] != "${address}"`).join(" && ");
+    return (`t=""; for f in ${tables.join(" ")}; do [ -r "$f" ] && t="$t $f"; done; ` +
+        `end=$((SECONDS+${seconds})); while :; do ok=1; for h in ${hex.join(" ")}; do ` +
+        `awk -v h="$h" '$4 == "0A" { split($2, a, ":"); if (a[2] == h && ${skip}) f = 1 } END { exit !f }' ` +
+        `$t </dev/null || ok=0; done; [ $ok = 1 ] && exit 0; ` +
+        `[ $SECONDS -ge $end ] && exit 1; sleep 0.2; done`);
+}
+/** Test hook: forget every process this module has seen, as a fresh client
+ * would. */
+export function resetProcesses() {
+    registries.clear();
+}
+/** Blaxel's status from Runtime's record. A process ended by a signal has a
+ * negative exit code on Runtime (-15 after SIGTERM, -9 after SIGKILL or its
+ * time limit): SIGKILL is killed, any other signal stopped, from any client. */
+function statusOf(state, exitCode) {
+    if (state === "running" && exitCode === null)
+        return "running";
+    if (exitCode === null)
+        return "killed";
+    if (exitCode < 0)
+        return exitCode === -SIGKILL ? "killed" : "stopped";
+    return exitCode === 0 ? "completed" : "failed";
+}
+/** Blaxel's restart notes in a process's output: the last is its restart count. */
+function restartsIn(logs) {
+    let count = 0;
+    for (const [, n] of logs.matchAll(/\n\[Process failed with exit code -?\d+\. Attempting restart (\d+)\//g))
+        count = Math.max(count, Number(n));
+    return count;
+}
+/** `sandbox.process`: run, follow and stop processes, by pid or by name. */
+export class SandboxProcess {
+    #ctx;
+    constructor(ctx) {
+        this.#ctx = ctx;
+    }
+    get #registry() {
+        return registryOf(this.#ctx.id);
+    }
+    #remember(id, meta, process) {
+        const registry = this.#registry;
+        registry.byName.set(meta.name, id);
+        registry.meta.set(id, meta);
+        if (process)
+            registry.processes.set(id, process);
+    }
+    /** Starts a command. Without waitForCompletion it answers at once, the
+     * process running; with it, once the process ends, with its output. */
+    async exec(request) {
+        const { onLog, onStdout, onStderr, ...req } = request;
+        if (typeof req.command !== "string" || req.command === "")
+            throw responseError(400, "command is required");
+        const env = req.env ?? {};
+        checkEnvNames(Object.keys(env));
+        const meta = {
+            name: req.name || randomName(),
+            command: req.command,
+            workingDir: req.workingDir ?? "",
+            ...(req.keepAlive ? { keepAlive: true } : {}),
+            ...(req.restartOnFailure
+                ? { restartOnFailure: true, maxRestarts: req.maxRestarts ?? 0 }
+                : {}),
+            ...(req.stdin ? { stdin: true } : {}),
+        };
+        const line = processLine(req.command, {
+            name: meta.name,
+            ...(req.restartOnFailure ? { maxRestarts: req.maxRestarts ?? 0 } : {}),
+            linkHome: /\/blaxel(\/|\s|$|["'`;:])/.test(`${req.command} ${Object.values(env).join(" ")}`),
+            ...(req.keepAlive ? { keepAlive: true } : {}),
+        });
+        const names = Object.keys(env);
+        const options = {
+            cwd: req.workingDir ? toRuntimePath(req.workingDir) : RUNTIME_HOME,
+            ...(names.length ? { env: { ...env, RUNTIME_BLAXEL_KEEP: `:${names.join(":")}:` } } : {}),
+            timeoutMs: this.#killAfterMs(req),
+        };
+        // Awake first: a sandbox that cannot be kept awake refuses the process
+        // rather than pausing under it.
+        if (req.keepAlive)
+            await this.#ctx.keepAwake(this.#killAfterMs(req) / 1000);
+        const callbacks = onLog || onStdout || onStderr;
+        if (req.waitForCompletion && !req.stdin && !req.waitForPorts?.length)
+            return await this.#execWaiting(line, options, meta, req.timeout, {
+                ...(onLog ? { onLog } : {}),
+                ...(onStdout ? { onStdout } : {}),
+                ...(onStderr ? { onStderr } : {}),
+            });
+        const runtime = await this.#ctx.live();
+        const process = await this.#retryPaused(() => runtime.spawn(line, { ...options, ...(req.stdin ? { stdin: "pipe" } : {}) }));
+        this.#remember(process.id, meta, process);
+        if (req.waitForPorts?.length)
+            await this.#waitForPorts(req.waitForPorts, req.timeout);
+        if (req.waitForCompletion) {
+            const bounded = req.timeout !== undefined && req.timeout > 0;
+            const done = await this.wait(process.id, {
+                maxWait: bounded ? req.timeout * 1000 : -1,
+            }).catch((error) => {
+                throw bounded && !(error instanceof ResponseError)
+                    ? responseError(422, `process timed out after ${req.timeout} seconds`)
+                    : error;
+            });
+            if (callbacks)
+                replay(done, { onLog, onStdout, onStderr });
+            return { ...done, close: () => undefined };
+        }
+        const running = this.#response(process.info, meta, { stdout: "", stderr: "", logs: "" });
+        if (!callbacks)
+            return running;
+        const stream = this.streamLogs(process.id, {
+            ...(onLog ? { onLog } : {}),
+            ...(onStdout ? { onStdout } : {}),
+            ...(onStderr ? { onStderr } : {}),
+        });
+        return { ...running, close: () => stream.close() };
+    }
+    #killAfterMs(req) {
+        if (!req.keepAlive)
+            return LONGEST_MS;
+        const seconds = req.timeout === undefined || req.timeout < 0 ? KEEP_ALIVE_SECONDS : req.timeout;
+        return seconds === 0 ? LONGEST_MS : Math.min(LONGEST_MS, seconds * 1000);
+    }
+    /** A call that found the sandbox paused (autoWake off) is made again once,
+     * after a wake: it never started, so it cannot run twice. */
+    async #retryPaused(work) {
+        try {
+            return await work();
+        }
+        catch (error) {
+            if (codeOf(error) !== "sandbox_paused")
+                throw translate(error);
+            await this.#ctx.wake();
+            return guard(work);
+        }
+    }
+    /** One streamed exec: the process's id arrives first, then its output, then
+     * its end. Past `timeout` seconds the wait gives up and the process keeps
+     * running, as in Blaxel. */
+    async #execWaiting(line, options, meta, timeout, callbacks) {
+        const runtime = await this.#ctx.live();
+        const abort = new AbortController();
+        let timedOut = false;
+        const timer = timeout && timeout > 0
+            ? setTimeout(() => {
+                timedOut = true;
+                abort.abort();
+            }, timeout * 1000)
+            : undefined;
+        const out = { stdout: "", stderr: "", logs: "" };
+        let id;
+        let exit;
+        const startedAt = new Date().toISOString();
+        const run = async () => {
+            for await (const event of runtime.execStream(line, { ...options, signal: abort.signal })) {
+                if (event.type === "start") {
+                    id = event.processId;
+                    this.#remember(id, meta);
+                }
+                else if (event.type === "stdout" || event.type === "stderr") {
+                    out[event.type] += event.data;
+                    out.logs += event.data;
+                    (event.type === "stdout" ? callbacks.onStdout : callbacks.onStderr)?.(event.data);
+                    callbacks.onLog?.(event.data);
+                }
+                else if (event.type === "exit")
+                    exit = event;
+            }
+        };
+        try {
+            try {
+                await run();
+            }
+            catch (error) {
+                if (timedOut)
+                    throw error;
+                if (id !== undefined || codeOf(error) !== "sandbox_paused")
+                    throw error;
+                await this.#ctx.wake();
+                await run();
+            }
+        }
+        catch (error) {
+            if (timedOut)
+                throw responseError(422, `process timed out after ${timeout} seconds`);
+            throw translate(error);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        const info = {
+            id: id ?? "",
+            kind: "process",
+            state: exit?.state ?? "exited",
+            exitCode: exit?.exitCode ?? null,
+            command: line,
+            cwd: options.cwd,
+            pty: false,
+            stdinOpen: false,
+            stdinOffset: 0,
+            startedAt,
+            endedAt: new Date().toISOString(),
+            timeoutMs: options.timeoutMs,
+            outputBytes: 0,
+            firstOffset: 0,
+        };
+        const done = await this.#settled(this.#response(info, meta, out));
+        return { ...done, close: () => abort.abort() };
+    }
+    /** Waits until every port listens on an address other than loopback, as
+     * Blaxel's waitForPorts, in one command inside the sandbox. */
+    async #waitForPorts(ports, timeout) {
+        const seconds = timeout && timeout > 0 ? timeout : 60;
+        for (const port of ports)
+            if (!Number.isInteger(port) || port < 1 || port > 65_535)
+                throw responseError(400, `invalid port ${port}`);
+        const script = portWaitScript(ports, seconds);
+        const runtime = await this.#ctx.live();
+        const result = await guard(() => runtime.exec(["bash", "-c", script], { timeoutMs: (seconds + 5) * 1000 }));
+        if (result.exitCode !== 0)
+            throw responseError(422, `process timed out waiting for ports after ${seconds} seconds`);
+    }
+    /** A keepAlive process seen to end gives the sandbox its idle pause back
+     * (when no other runs) before the answer. */
+    async #settled(response) {
+        if (response.keepAlive && response.status !== "running")
+            await this.#ctx.keepAliveEnded();
+        return response;
+    }
+    #response(info, meta, out) {
+        const parsed = meta ?? metaOf(info);
+        const status = statusOf(info.state, info.exitCode);
+        return {
+            command: parsed.command,
+            completedAt: status === "running" ? "" : httpDate(info.endedAt ?? Date.now()),
+            exitCode: info.exitCode === null ? (status === "running" ? 0 : -1) : Math.max(-1, info.exitCode),
+            logs: out.logs,
+            name: parsed.name,
+            pid: info.id,
+            startedAt: httpDate(info.startedAt),
+            status,
+            stderr: out.stderr,
+            stdout: out.stdout,
+            workingDir: parsed.workingDir,
+            keepAlive: parsed.keepAlive ?? false,
+            restartOnFailure: parsed.restartOnFailure ?? false,
+            maxRestarts: parsed.maxRestarts ?? 0,
+            restartCount: restartsIn(out.logs),
+            stdin: parsed.stdin ?? false,
+        };
+    }
+    /** A process by pid or name: the registry first, then Runtime's list. */
+    async #find(identifier) {
+        const runtime = await this.#ctx.live();
+        const registry = this.#registry;
+        const known = registry.byName.get(identifier) ?? (registry.meta.has(identifier) ? identifier : undefined);
+        if (known !== undefined) {
+            const process = await this.#retryPaused(() => runtime.processes.get(known));
+            registry.processes.set(known, process);
+            return { id: known, info: process.info, process };
+        }
+        const all = await this.#retryPaused(() => runtime.processes.list());
+        let found;
+        for (const info of all) {
+            const parsed = parseProcessLine(info.command);
+            const matches = parsed?.name === identifier || info.id === identifier;
+            if (matches && (!found || Date.parse(info.startedAt) >= Date.parse(found.startedAt)))
+                found = info;
+        }
+        if (!found)
+            throw responseError(404, "process not found");
+        const parsed = parseProcessLine(found.command);
+        if (parsed)
+            this.#remember(found.id, { ...metaOf(found), name: parsed.name });
+        return { id: found.id, info: found };
+    }
+    async #process(found) {
+        if (found.process)
+            return found.process;
+        const cached = this.#registry.processes.get(found.id);
+        if (cached)
+            return cached;
+        const runtime = await this.#ctx.live();
+        const process = await guard(() => runtime.processes.get(found.id));
+        this.#registry.processes.set(found.id, process);
+        return process;
+    }
+    /** The output so far: to the end for a process that has ended, to what it
+     * had written when read for one still running. */
+    async #output(found, untilExit, signal) {
+        const out = { stdout: "", stderr: "", logs: "" };
+        const running = found.info.state === "running";
+        if (running && !untilExit && found.info.outputBytes <= found.info.firstOffset)
+            return out;
+        const runtime = await this.#ctx.live();
+        const abort = new AbortController();
+        const forward = () => abort.abort();
+        signal?.addEventListener("abort", forward, { once: true });
+        try {
+            for await (const event of runtime.processes.follow(found.id, {
+                cursor: 0,
+                signal: abort.signal,
+            })) {
+                if (event.type === "stdout" || event.type === "stderr") {
+                    out[event.type] += event.data;
+                    out.logs += event.data;
+                    const reached = event.offset + Buffer.byteLength(event.data);
+                    if (running && !untilExit && reached >= found.info.outputBytes)
+                        break;
+                }
+                else if (event.type === "exit") {
+                    found.info = {
+                        ...found.info,
+                        state: event.state,
+                        exitCode: event.exitCode,
+                        endedAt: found.info.endedAt ?? new Date().toISOString(),
+                    };
+                    break;
+                }
+            }
+            // A follow cut short by the caller's signal ends without its exit.
+            signal?.throwIfAborted();
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw signal.reason ?? error;
+            throw translate(error);
+        }
+        finally {
+            abort.abort();
+            signal?.removeEventListener("abort", forward);
+        }
+        return out;
+    }
+    /** The process, with the output it has written so far. */
+    async get(identifier, _options = {}) {
+        const found = await this.#find(identifier);
+        const out = await this.#output(found, false);
+        return this.#settled(this.#response(found.info, this.#registry.meta.get(found.id), out));
+    }
+    /** Waits for the process to end (at most `maxWait` ms, -1 for no limit) and
+     * answers it with its output. The process is never stopped by the wait. */
+    async wait(identifier, { maxWait = 60_000, interval = 1000, signal, } = {}) {
+        if (!Number.isFinite(maxWait) ||
+            (maxWait < 0 && maxWait !== -1) ||
+            !Number.isFinite(interval) ||
+            interval <= 0)
+            throw new RangeError("maxWait must be -1 or finite and non-negative; interval must be finite and positive");
+        signal?.throwIfAborted();
+        const late = () => new Error(`Process did not finish in time (${identifier}); it may still be running`);
+        if (maxWait === 0)
+            throw late();
+        const abort = new AbortController();
+        const forward = () => abort.abort(signal?.reason);
+        signal?.addEventListener("abort", forward, { once: true });
+        const timer = maxWait === -1 ? undefined : setTimeout(() => abort.abort(late()), maxWait);
+        try {
+            const found = await this.#find(identifier);
+            const out = await this.#output(found, true, abort.signal);
+            return await this.#settled(this.#response(found.info, this.#registry.meta.get(found.id), out));
+        }
+        catch (error) {
+            if (abort.signal.aborted)
+                throw abort.signal.reason ?? error;
+            throw error;
+        }
+        finally {
+            clearTimeout(timer);
+            signal?.removeEventListener("abort", forward);
+        }
+    }
+    /** Every process started through this adapter, running or ended. Their
+     * output is left empty: read one with get() or logs(). */
+    async list() {
+        const runtime = await this.#ctx.live();
+        const all = await this.#retryPaused(() => runtime.processes.list());
+        const registry = this.#registry;
+        return all
+            .filter((info) => registry.meta.has(info.id) || parseProcessLine(info.command))
+            .map((info) => this.#response(info, registry.meta.get(info.id), { stdout: "", stderr: "", logs: "" }));
+    }
+    /** SIGTERM. */
+    async stop(identifier) {
+        await this.#signal(identifier, "SIGTERM");
+        return { message: "Process stop requested" };
+    }
+    /** SIGKILL. */
+    async kill(identifier) {
+        await this.#signal(identifier, "SIGKILL");
+        return { message: "Process kill requested" };
+    }
+    /** Signals the process. A keepAlive one is given up to 2 s to end, and then
+     * the sandbox gets its idle pause back when no other runs. */
+    async #signal(identifier, signal) {
+        const found = await this.#find(identifier);
+        const process = await this.#process(found);
+        await guard(() => process.kill(signal));
+        const meta = this.#registry.meta.get(found.id) ?? metaOf(found.info);
+        if (!meta.keepAlive)
+            return;
+        const abort = new AbortController();
+        const timer = setTimeout(() => abort.abort(), 2000);
+        try {
+            await this.#output({ ...found, info: { ...found.info, state: "running" } }, true, abort.signal);
+        }
+        catch {
+            // Still ending after 2 s: the next check gives the idle pause back.
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        await this.#ctx.keepAliveEnded();
+    }
+    /** Bytes to the standard input of a process started with `stdin: true`,
+     * sent as they are. */
+    async writeStdin(identifier, data) {
+        const process = await this.#process(await this.#find(identifier));
+        await guard(() => process.write(data));
+    }
+    /** Closes the process's standard input (EOF). */
+    async closeStdin(identifier) {
+        const process = await this.#process(await this.#find(identifier));
+        await guard(() => process.write("", { eof: true }));
+    }
+    /** The output so far: both streams in the order written ("all"), or one. */
+    async logs(identifier, type = "all") {
+        if (type !== "all" && type !== "stdout" && type !== "stderr")
+            throw new Error("Unsupported log type");
+        const found = await this.#find(identifier);
+        const out = await this.#output(found, false);
+        return type === "all" ? out.logs : out[type];
+    }
+    /** Follows the output from the start until the process ends, a line at a
+     * time, as Blaxel's streamLogs. `close()` stops following. */
+    streamLogs(identifier, options = {}) {
+        const abort = new AbortController();
+        const partial = { stdout: "", stderr: "" };
+        const emit = (stream, line) => {
+            (stream === "stdout" ? options.onStdout : options.onStderr)?.(line);
+            options.onLog?.(line);
+        };
+        const done = (async () => {
+            try {
+                const found = await this.#find(identifier);
+                const runtime = await this.#ctx.live();
+                for await (const event of runtime.processes.follow(found.id, {
+                    cursor: 0,
+                    signal: abort.signal,
+                })) {
+                    if (event.type === "stdout" || event.type === "stderr") {
+                        const lines = (partial[event.type] + event.data).split(/\r?\n/);
+                        partial[event.type] = lines.pop();
+                        for (const line of lines)
+                            emit(event.type, line);
+                    }
+                    else if (event.type === "exit")
+                        break;
+                }
+            }
+            catch (error) {
+                if (abort.signal.aborted)
+                    return;
+                const failure = translate(error);
+                const asError = failure instanceof Error ? failure : new Error(String(failure));
+                options.onError?.(asError);
+                throw asError;
+            }
+            finally {
+                for (const stream of ["stdout", "stderr"])
+                    if (partial[stream].trim())
+                        emit(stream, partial[stream]);
+            }
+        })();
+        void done.catch(() => undefined);
+        return { close: () => abort.abort(), wait: () => done };
+    }
+}
+/** What a process's recorded command says about it, when this adapter did
+ * not start it in this client. */
+function metaOf(info) {
+    const parsed = parseProcessLine(info.command);
+    return {
+        name: parsed?.name ?? info.id,
+        command: parsed?.command ?? info.command,
+        workingDir: info.cwd === RUNTIME_HOME ? "" : toBlaxelPath(info.cwd),
+        ...(parsed?.keepAlive ? { keepAlive: true } : {}),
+    };
+}
+/** Output a waited process already wrote, given to the callbacks line by line. */
+function replay(done, callbacks) {
+    for (const line of done.stdout.split("\n").filter(Boolean))
+        callbacks.onStdout?.(line);
+    for (const line of done.stderr.split("\n").filter(Boolean))
+        callbacks.onStderr?.(line);
+    for (const line of done.logs.split("\n").filter(Boolean))
+        callbacks.onLog?.(line);
+}