promigence 0.1.0-beta.12

package/dist/device.js

@@ -0,0 +1,129 @@
+import { execFileSync } from "node:child_process";
+import { createHash, randomBytes } from "node:crypto";
+import { chmodSync, closeSync, constants, lstatSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeSync, } from "node:fs";
+import { basename, dirname, join, win32 } from "node:path";
+import { configDir } from "./sdk/index.js";
+export const DEVICE_HEADER = "promigence-device";
+const HEX64 = /^[0-9a-f]{64}$/;
+export function parseIoregUuid(out) {
+    return /"IOPlatformUUID"\s*=\s*"([^"\r\n]+)"/.exec(out)?.[1];
+}
+export function parseRegMachineGuid(out) {
+    return /MachineGuid\s+REG_SZ\s+(\S+)/i.exec(out)?.[1];
+}
+const realIo = {
+    run: (cmd, args) => execFileSync(cmd, args, {
+        encoding: "utf8", timeout: 2_000, windowsHide: true, stdio: ["ignore", "pipe", "ignore"], maxBuffer: 1 << 20,
+    }),
+    read: (path) => readFileSync(path, "utf8"),
+    env: process.env,
+};
+function system32(name, env) {
+    const root = env["SystemRoot"] ?? env["SYSTEMROOT"];
+    return win32.join(root && /^[A-Za-z]:[\\/]/.test(root) ? root : "C:\\Windows", "System32", name);
+}
+export function rawMachineId(platform = process.platform, io = realIo) {
+    const tryRead = (f) => {
+        try {
+            const v = f()?.trim();
+            return v ? v : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    };
+    if (platform === "darwin")
+        return tryRead(() => parseIoregUuid(io.run("/usr/sbin/ioreg", ["-rd1", "-c", "IOPlatformExpertDevice"])));
+    if (platform === "win32") {
+        return tryRead(() => parseRegMachineGuid(io.run(system32("reg.exe", io.env), [
+            "query", "HKLM\\SOFTWARE\\Microsoft\\Cryptography", "/v", "MachineGuid",
+        ])));
+    }
+    return tryRead(() => io.read("/etc/machine-id")) ?? tryRead(() => io.read("/var/lib/dbus/machine-id"));
+}
+export function machineHash(raw) {
+    if (!raw)
+        return undefined;
+    return createHash("sha256").update(raw.trim().toLowerCase()).digest("hex");
+}
+export function installIdPath(env = process.env) {
+    return join(configDir(env), "install-id");
+}
+function readInstallId(path) {
+    const st = lstatSync(path, { throwIfNoEntry: false });
+    if (!st || !st.isFile())
+        return undefined;
+    const v = readFileSync(path, "utf8").trim();
+    return HEX64.test(v) ? v : undefined;
+}
+export function installId(env = process.env) {
+    const path = installIdPath(env);
+    try {
+        const have = readInstallId(path);
+        if (have)
+            return have;
+        if (lstatSync(path, { throwIfNoEntry: false })?.isSymbolicLink())
+            return undefined;
+        const dir = dirname(path);
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+        const dst = statSync(dir);
+        const uid = typeof process.getuid === "function" ? process.getuid() : undefined;
+        if (!dst.isDirectory() || (uid !== undefined && dst.uid !== uid))
+            return undefined;
+        const id = randomBytes(32).toString("hex");
+        const tmp = join(dir, `.${basename(path)}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
+        const fd = openSync(tmp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW ?? 0), 0o600);
+        try {
+            writeSync(fd, id + "\n");
+        }
+        finally {
+            closeSync(fd);
+        }
+        try {
+            chmodSync(tmp, 0o600);
+            renameSync(tmp, path);
+        }
+        catch {
+            try {
+                unlinkSync(tmp);
+            }
+            catch { }
+            return undefined;
+        }
+        return readInstallId(path) ?? id;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function deviceHeaderValue(machine, install) {
+    const parts = [];
+    if (machine && HEX64.test(machine))
+        parts.push(`m=${machine}`);
+    if (install && HEX64.test(install))
+        parts.push(`i=${install}`);
+    return parts.length ? parts.join("; ") : undefined;
+}
+let cached;
+export function deviceHeader() {
+    if (!cached) {
+        let v;
+        try {
+            v = deviceHeaderValue(machineHash(rawMachineId()), installId());
+        }
+        catch {
+            v = undefined;
+        }
+        cached = { v };
+    }
+    return cached.v;
+}
+export function deviceFetch(value, base = globalThis.fetch) {
+    if (!value)
+        return undefined;
+    return ((input, init) => {
+        const headers = new Headers(init?.headers);
+        headers.set(DEVICE_HEADER, value);
+        return base(input, { ...init, headers });
+    });
+}

package/dist/dirsnap.js

@@ -0,0 +1,378 @@
+import { spawnSync } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { closeSync, fstatSync, lstatSync, mkdtempSync, openSync, readSync, realpathSync, rmSync, statSync, writeFileSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, join, resolve, sep } from "node:path";
+import { DEFAULT_SNAPSHOT, FILE_PUT_CHUNK_BYTES, PromigenceError, secretLike } from "./sdk/index.js";
+import { bool, client, list, nexts, str, waitBound } from "./commands.js";
+import { emit, okEnvelope, parseDuration, usageError, write } from "./io.js";
+import { jsonLine, sh } from "./term.js";
+import { CREDENTIAL_REMEDY, confirmOverride, credentialRefusal, emitUploadList, localTarNoMetadataArgs } from "./uploadguard.js";
+const BASE_WORKSPACE = "/work";
+export const DIR_SANDBOX_TIMEOUT_DEFAULT = "30m";
+const STEP_TIMEOUT_MS = 30 * 60_000;
+const STEP_OUTPUT_BYTES = 256 * 1024;
+const FAILED_TAIL_LINES = 20;
+const PROGRESS_FROM_BYTES = 8 * 1024 * 1024;
+const INTERRUPT_START_WAIT_MS = 15_000;
+const INTERRUPTED = {
+    ended: () => "The build sandbox was ended.",
+    kept: (id) => `The build sandbox was kept (--keep-sandbox); end it with ${sh `promigence run kill ${id ?? "<run_id>"}`}.`,
+    none: () => "No build sandbox was started.",
+    unknown: (id) => id
+        ? `The build sandbox may still be running; end it with ${sh `promigence run kill ${id}`}.`
+        : "A build sandbox may still be starting; `promigence run list` shows it and `promigence run kill <run_id>` ends it.",
+};
+const NOT_WITH_DIR = [
+    "--repo", "--commit", "--ref", "--track", "--also-repo", "--image", "--dockerfile", "--context",
+    "--devcontainer", "--start", "--grade", "--holdout", "--grade-score", "--grade-timeout",
+    "--grade-output", "--tier",
+];
+export { secretLike };
+function git(dir, args) {
+    const r = spawnSync("git", ["-C", dir, ...args], { encoding: "buffer", maxBuffer: 512 * 1024 * 1024 });
+    if (r.error && r.error.code === "ENOENT")
+        return { ok: false, out: "", err: "", missing: true };
+    return { ok: r.status === 0, out: r.stdout?.toString("utf8") ?? "", err: r.stderr?.toString("utf8") ?? "" };
+}
+export function listTreeFiles(dir, opts = {}) {
+    const inside = git(dir, ["rev-parse", "--is-inside-work-tree"]);
+    if (inside.missing) {
+        throw usageError("`git` is not on PATH, and --dir lists what it uploads with git", "Install git, or upload a directory into a sandbox with `promigence files put DIR DST --recursive`.");
+    }
+    if (!inside.ok || inside.out.trim() !== "true") {
+        throw usageError(`${dir} is not inside a git work tree: --dir uploads the files git tracks there`, "Run `git init && git add -A` in it first (no commit needed), or point --dir at your repository. --include-untracked does not help here: it adds the untracked files of a work tree.");
+    }
+    const tracked = git(dir, ["ls-files", "-z", "--cached", "--recurse-submodules"]);
+    if (!tracked.ok)
+        throw usageError(`git ls-files failed in ${dir}: ${tracked.err.trim()}`, "Check the repository with `git status`.");
+    const seen = new Set();
+    const files = [];
+    const outside = [];
+    let missing = 0;
+    let bytes = 0;
+    const realRoot = realpathSync(dir);
+    const linkedDirs = new Map();
+    const throughLink = (p) => {
+        const parts = p.split("/");
+        for (let i = 1; i < parts.length; i++) {
+            const d = parts.slice(0, i).join("/");
+            let linked = linkedDirs.get(d);
+            if (linked === undefined) {
+                try {
+                    linked = !lstatSync(join(dir, d)).isDirectory();
+                }
+                catch {
+                    linked = false;
+                }
+                linkedDirs.set(d, linked);
+            }
+            if (linked)
+                return true;
+        }
+        return false;
+    };
+    const take = (p) => {
+        if (!p || seen.has(p))
+            return false;
+        seen.add(p);
+        if (throughLink(p)) {
+            outside.push(p);
+            return false;
+        }
+        let st;
+        try {
+            st = lstatSync(join(dir, p));
+        }
+        catch {
+            missing += 1;
+            return false;
+        }
+        if (st.isDirectory())
+            return false;
+        if (!st.isSymbolicLink()) {
+            let real;
+            try {
+                real = realpathSync(join(dir, p));
+            }
+            catch {
+                missing += 1;
+                return false;
+            }
+            if (real !== realRoot && !real.startsWith(realRoot + sep)) {
+                outside.push(p);
+                return false;
+            }
+        }
+        files.push(p);
+        if (st.isFile())
+            bytes += st.size;
+        return true;
+    };
+    for (const p of tracked.out.split("\0"))
+        take(p);
+    let untracked = 0;
+    if (opts.includeUntracked) {
+        const others = git(dir, ["ls-files", "-z", "--others", "--exclude-standard"]);
+        if (!others.ok)
+            throw usageError(`git ls-files failed in ${dir}: ${others.err.trim()}`, "Check the repository with `git status`.");
+        for (const p of others.out.split("\0"))
+            if (take(p))
+                untracked += 1;
+    }
+    files.sort();
+    outside.sort();
+    return { files, missing, untracked, bytes, outside };
+}
+export function refuseSecrets(files, allow) {
+    const hits = files.filter(secretLike);
+    if (hits.length && !allow)
+        throw credentialRefusal(hits, CREDENTIAL_REMEDY.dir);
+    return hits;
+}
+export function human(n) {
+    if (n < 1024)
+        return `${n} B`;
+    const u = ["KiB", "MiB", "GiB", "TiB"];
+    let v = n / 1024;
+    let i = 0;
+    while (v >= 1024 && i < u.length - 1) {
+        v /= 1024;
+        i++;
+    }
+    return `${v.toFixed(v < 10 ? 1 : 0)} ${u[i]}`;
+}
+export function packFiles(dir, files, out) {
+    const listPath = `${out}.list`;
+    writeFileSync(listPath, files.join("\0") + (files.length ? "\0" : ""));
+    try {
+        const r = spawnSync("tar", ["-c", "-f", out, ...localTarNoMetadataArgs(), "-C", dir, "--no-recursion", "--null", "-T", listPath], {
+            encoding: "utf8", env: { ...process.env, COPYFILE_DISABLE: "1" },
+        });
+        if (r.error && r.error.code === "ENOENT") {
+            throw usageError("`tar` is not on PATH, and --dir packs the tree with it", "Install tar (macOS and every Linux ship one).");
+        }
+        if (r.status !== 0) {
+            throw usageError(`packing ${dir} failed: ${(r.stderr ?? "").trim() || `tar exited ${r.status}`}`, "Check that every listed file is readable (`git status`).");
+        }
+    }
+    finally {
+        rmSync(listPath, { force: true });
+    }
+}
+function tail(r) {
+    const lines = `${r.stdout ?? ""}${r.stderr ? `\n${r.stderr}` : ""}`.split("\n").filter((l) => l.trim() !== "");
+    return lines.slice(-FAILED_TAIL_LINES);
+}
+const exitOf = (r) => r.timed_out ? null : typeof r.exit_code === "number" ? r.exit_code : 1;
+export async function dirSnapshotCreate(ctx) {
+    for (const f of NOT_WITH_DIR) {
+        if (bool(ctx, f)) {
+            throw usageError(`${f} is not taken with --dir`, f === "--tier"
+                ? "The snapshot has the size of the snapshot the upload starts from: pass --from a snapshot built at that tier."
+                : `--dir uploads a local tree into a sandbox of --from (default ${DEFAULT_SNAPSHOT}); drop ${f}, or build that form without --dir.`);
+        }
+    }
+    const dirArg = str(ctx, "--dir");
+    if (dirArg === undefined)
+        throw usageError("--dir needs a directory", "Example: promigence snapshot create --dir . --alias app --wait.");
+    const dir = resolve(dirArg);
+    try {
+        if (!statSync(dir).isDirectory())
+            throw new Error("not a directory");
+    }
+    catch {
+        throw usageError(`--dir ${dirArg} is not a directory`, "Pass the path of your working tree: --dir .");
+    }
+    const listing = listTreeFiles(dir, { includeUntracked: bool(ctx, "--include-untracked") });
+    if (!listing.files.length) {
+        throw usageError(`nothing to upload: git tracks no file in ${dirArg}`, "`git add` the files first, or pass --include-untracked for files git does not ignore.");
+    }
+    if (bool(ctx, "--list-files")) {
+        return emitUploadList(ctx, {
+            kind: "dir", root: dirArg, paths: listing.files, files: listing.files.length, bytes: listing.bytes,
+            secrets: listing.files.filter(secretLike),
+            notes: listing.outside.length
+                ? [`Left out ${listing.outside.length} path${listing.outside.length === 1 ? "" : "s"} under a directory that is now a link or outside ${dirArg}: ${listing.outside.slice(0, 10).join(", ")}${listing.outside.length > 10 ? ` and ${listing.outside.length - 10} more` : ""}.`]
+                : [],
+        });
+    }
+    const allowed = await confirmOverride(ctx, {
+        flag: "--allow-secrets", what: "files that look like credentials",
+        items: refuseSecrets(listing.files, bool(ctx, "--allow-secrets")),
+    });
+    const from = str(ctx, "--from") ?? DEFAULT_SNAPSHOT;
+    const alias = str(ctx, "--alias");
+    const setup = list(ctx, "--setup");
+    const verify = str(ctx, "--verify");
+    const keep = bool(ctx, "--keep-sandbox");
+    const timeoutMs = parseDuration(str(ctx, "--timeout") ?? DIR_SANDBOX_TIMEOUT_DEFAULT, "--timeout");
+    if (timeoutMs <= 0)
+        throw usageError("--timeout must be more than 0", `Omit it for ${DIR_SANDBOX_TIMEOUT_DEFAULT}.`);
+    const c = client(ctx);
+    const q = await c.runs.quote({ snapshot: from, count: 1, timeoutMs });
+    const perMin = q.per_second_usd * 60;
+    const most = q.quote?.max_cost ?? q.max_cost;
+    const priceFields = {
+        per_second_usd: q.per_second_usd, per_minute_usd: Number(perMin.toFixed(6)), tier: q.tier,
+        max_cost: q.max_cost, sandbox_timeout_s: Math.ceil(timeoutMs / 1000), snapshot_charge: 0,
+    };
+    await write(process.stderr, ctx.mode === "json"
+        ? jsonLine(process.stderr, { hint: "price", ...priceFields, unit: "USD" })
+        : `price: the sandbox is billed per second while the upload and setup run (~$${perMin.toFixed(4)}/min at tier ${q.tier}, at most $${q.max_cost.toFixed(2)} for its ${Math.round(timeoutMs / 60_000)} min limit); the snapshot itself is not charged\n` +
+            `  uploading ${listing.files.length} file${listing.files.length === 1 ? "" : "s"} (${human(listing.bytes)}) from ${dirArg}${listing.untracked ? `, ${listing.untracked} untracked` : ""}${listing.missing ? `; ${listing.missing} deleted in the working tree, skipped` : ""}${allowed.length ? `; ${allowed.length} secret-looking, allowed by --allow-secrets` : ""}\n` +
+            (listing.outside.length ? `  skipped ${listing.outside.length} under a directory that is now a link or outside ${dirArg}: ${listing.outside.slice(0, 10).join(", ")}${listing.outside.length > 10 ? " ..." : ""}\n` : ""));
+    const scratchDir = mkdtempSync(join(tmpdir(), "promigence-dir-"));
+    const tarPath = join(scratchDir, "tree.tar");
+    let runId;
+    let starting;
+    let ended = false;
+    const t0 = Date.now();
+    const end = async () => {
+        if (runId === undefined && starting !== undefined) {
+            const got = await Promise.race([
+                starting.then((id) => id, () => null),
+                new Promise((r) => setTimeout(() => r(undefined), INTERRUPT_START_WAIT_MS).unref()),
+            ]);
+            if (got === undefined)
+                return "unknown";
+            if (got !== null)
+                runId = got;
+        }
+        if (runId === undefined)
+            return "none";
+        if (keep)
+            return "kept";
+        if (ended)
+            return "ended";
+        ended = true;
+        return c.runs.kill(runId).then(() => "ended", () => "unknown");
+    };
+    const onSigint = () => {
+        void end().then((what) => {
+            rmSync(scratchDir, { recursive: true, force: true });
+            process.stderr.write(`\npromigence: interrupted; nothing was snapshotted. ${INTERRUPTED[what](runId)}\n`);
+            process.exit(130);
+        });
+    };
+    process.once("SIGINT", onSigint);
+    let result;
+    try {
+        packFiles(dir, listing.files, tarPath);
+        const packed = statSync(tarPath).size;
+        const spendCap = Math.ceil((most + 0.01) * 100) / 100;
+        starting = c.runs.start({ snapshot: from, count: 1, timeoutMs, spendCap }).then((s) => s.run_id);
+        runId = await starting;
+        if (keep)
+            ctx.startedRunId = runId;
+        const waited = await c.runs.waitReady(runId, { waitTimeoutMs: waitBound(ctx) });
+        const sb = waited.sandboxes.find((s) => s.state === "ready" || s.state === "running");
+        if (waited.timedOut || !sb) {
+            throw new PromigenceError({
+                code: waited.timedOut ? "timeout" : "fork_failed",
+                message: waited.timedOut
+                    ? `the sandbox did not start within the wait (--wait-timeout); it was ended and nothing was uploaded`
+                    : `the sandbox ended before it started (${waited.sandboxes.map((s) => s.state).join(", ")}); nothing was uploaded`,
+                fix: "Run the command again; `promigence status` shows whether capacity is short.",
+                exitCode: waited.timedOut ? 11 : 3,
+            });
+        }
+        const sid = sb.sandbox_id;
+        let workspace = BASE_WORKSPACE;
+        if (from !== DEFAULT_SNAPSHOT) {
+            const where = await c.sandboxes.exec(sid, { sh: "pwd", timeoutMs: 60_000 });
+            const w = where.stdout.trim().split("\n")[0] ?? "";
+            if (exitOf(where) === 0 && w.startsWith("/") && w !== "/")
+                workspace = w;
+        }
+        const scratch = `/tmp/.promigence-upload/dir-${Date.now().toString(36)}-${randomBytes(6).toString("hex")}.tar`;
+        const fd = openSync(tarPath, "r");
+        try {
+            const size = fstatSync(fd).size;
+            const buf = Buffer.alloc(FILE_PUT_CHUNK_BYTES);
+            let lastSaid = Date.now();
+            const show = size > PROGRESS_FROM_BYTES && ctx.mode !== "json";
+            await c.sandboxes.files.putFrom(sid, scratch, {
+                size,
+                read: (offset, len) => {
+                    const want = Math.min(len, buf.length);
+                    let got = 0;
+                    while (got < want) {
+                        const n = readSync(fd, buf, got, want - got, offset + got);
+                        if (n === 0)
+                            break;
+                        got += n;
+                    }
+                    if (show && Date.now() - lastSaid >= 2000) {
+                        lastSaid = Date.now();
+                        process.stderr.write(`  uploading ${human(offset)} / ${human(size)}\n`);
+                    }
+                    return buf.subarray(0, got);
+                },
+            });
+            if (ctx.mode === "json" && size > PROGRESS_FROM_BYTES) {
+                await write(process.stderr, jsonLine(process.stderr, { hint: "uploaded", bytes: size }));
+            }
+        }
+        finally {
+            closeSync(fd);
+        }
+        try {
+            await c.sandboxes.files.extract(sid, scratch, workspace);
+        }
+        finally {
+            await c.sandboxes.files.delete(sid, scratch).catch(() => { });
+        }
+        const steps = [
+            ...setup.map((s, i) => [`setup step ${i + 1}`, s]),
+            ...(verify === undefined ? [] : [["verify", verify]]),
+        ];
+        for (const [what, cmd] of steps) {
+            if (ctx.mode !== "json")
+                await write(process.stderr, `  $ ${cmd}\n`);
+            const r = await c.sandboxes.exec(sid, { sh: cmd, cwd: workspace, timeoutMs: STEP_TIMEOUT_MS, maxOutputBytes: STEP_OUTPUT_BYTES });
+            const code = exitOf(r);
+            if (code === 0)
+                continue;
+            const lines = tail(r);
+            await write(process.stderr, ctx.mode === "json"
+                ? jsonLine(process.stderr, { hint: "step_failed", step: what, cmd, exit_code: code, timed_out: code === null, tail: lines })
+                : lines.map((l) => `  | ${l}`).join("\n") + (lines.length ? "\n" : ""));
+            throw new PromigenceError({
+                code: "nonzero_exit",
+                message: `${what} (\`${cmd}\`) ${code === null ? "did not finish within 30 min" : `exited ${code}`}; nothing was snapshotted and the sandbox was ended`,
+                fix: "Fix the step (its last lines are above) and run the command again; to look around first, add --keep-sandbox and `promigence exec --sandbox <id> -- sh`.",
+                exitCode: 4,
+            });
+        }
+        const snap = await c.sandboxes.snapshot(sid, { ...(alias === undefined ? {} : { alias, moveAlias: true }) });
+        result = { snapshot_id: snap.snapshot_id, alias: snap.alias ?? alias ?? null, sandbox_id: sid, workspace, packed };
+    }
+    finally {
+        process.removeListener("SIGINT", onSigint);
+        await end();
+        rmSync(scratchDir, { recursive: true, force: true });
+    }
+    const seconds = Math.round((Date.now() - t0) / 100) / 10;
+    const name = result.alias ?? result.snapshot_id;
+    const next = nexts([sh `promigence run --snapshot ${name} --count 1 --cap 1 -- ls`, "start it and run a command where the tree is"], [sh `promigence snapshot inspect ${result.snapshot_id}`, "what was recorded"]);
+    return emit(ctx.mode, okEnvelope(ctx.cmd.name, {
+        snapshot_id: result.snapshot_id,
+        alias: result.alias,
+        source: { kind: "dir", dir: dirArg, files: listing.files.length, untracked: listing.untracked, bytes: listing.bytes, packed_bytes: result.packed },
+        ...(allowed.length ? { secret_files_allowed: allowed } : {}),
+        ...(listing.outside.length ? { skipped_outside: listing.outside } : {}),
+        from,
+        workspace: result.workspace,
+        setup,
+        ...(verify === undefined ? {} : { verify }),
+        sandbox_id: result.sandbox_id,
+        sandbox_ended: !keep,
+        sandbox_seconds: seconds,
+        price: priceFields,
+    }, next), [
+        `snapshot ${result.snapshot_id}${result.alias ? ` (${result.alias})` : ""}: ${listing.files.length} files from ${basename(dir)} in ${result.workspace}${setup.length ? `, ${setup.length} setup step${setup.length === 1 ? "" : "s"}` : ""}${verify ? ", verified" : ""}`,
+        `  sandbox ${result.sandbox_id}: ${seconds}s, ${keep ? `kept (--keep-sandbox): kill it with ${sh `promigence sandbox kill ${result.sandbox_id}`}` : "ended"}`,
+        `  next: ${next[0].cmd}`,
+    ], 0, ctx.pointer);
+}