sandboxedjs 0.1.72 → 0.1.74

package/README.md

@@ -22,6 +22,27 @@ await box.exec("python3 -c 'print(2**64)'"); // → 18446744073709551616
 box.dispose();
 ```
 
+## Python packages with compiled extensions
+
+`pip install` works for packages with C, Cython, Rust or Meson extensions. A
+wheel built for this runtime is used when one exists; otherwise one is built
+from source, from a recipe generated out of what PyPI and the package already
+state.
+
+```js
+configurePython({ buildFromSource: true });                          // build here (Node)
+configurePython({ buildFromSource: "http://localhost:4180/build" }); // ask a builder
+```
+
+Browsers have no compiler, so they ask a machine that does:
+
+```bash
+npx sandboxedjs-build-wheels 4180
+```
+
+See `docs/python/build-on-miss.md`, and `docs/python/compatibility.md` for what
+has been built and tested.
+
 ## Why
 
 Sometimes you need to run untrusted or generated code, give an AI agent a shell, build a
@@ -144,6 +165,17 @@ await session.run("echo $TOKEN in $(pwd)"); // → abc in /app
 
 ### Long-running processes
 
+A job put in the background with `&` keeps running after the command that started it returns,
+even from a stateless `exec`, and stops with `kill %N` in a session or when the container is
+disposed:
+
+```ts
+await box.exec("node server.js > /tmp/server.log 2>&1 &", { cwd: "/app" });
+await box.waitForPort(3000);
+```
+
+To hold the process yourself, spawn it:
+
 ```ts
 const proc = box.spawn("node server.js", { cwd: "/app" });
 
@@ -524,7 +556,10 @@ The container has no access to your filesystem, environment, or network unless y
 - The filesystem is entirely in memory. Code inside cannot read or write a host path — there is
   no `/Users`, no `/home/you`, no way to reach one.
 - Outbound network access is **off by default**; `curl https://…` fails until you pass
-  `network: { allowOutbound: true }`, optionally narrowed with `allowedHosts`.
+  `network: { allowOutbound: true }`, optionally narrowed with `allowedHosts`. The same policy
+  binds a program's own `fetch`, `http`, `https` and `WebSocket`: a refused request fails with
+  `ENETUNREACH`. `localhost` and `127.0.0.1` always mean the container's own servers — never the
+  host's.
 - Host files enter only through `files`, `mount()` or `copyIn()`, and leave only through
   `copyOut()` or `fs.readFile()`.
 - `timeoutMs` bounds runaway commands, and `exec` settles even when a process ignores its kill
@@ -979,7 +1014,16 @@ Honest list of what does not work:
   through is to answer **No** to a prompt like `npm create vite`'s "Install with npm and start
   now?" and run `npm install && npm run dev` from the shell instead.
 - **No `net`, `tls`, `worker_threads` or `vm`.** `http` and `https` are served by a virtual stack
-  that `request()` talks to directly, so servers work; raw sockets do not.
+  that `request()` talks to directly, so servers work; raw sockets do not. A program can reach
+  servers anywhere in the container over HTTP (`http.get`, `fetch`), but not open a `WebSocket` to
+  one.
+- **`node:test` covers what test files use** — `test`/`it`, `describe`, hooks, subtests, `skip`,
+  `todo`, `only`, `mock.fn` and `mock.method`, with `spec` and `tap` reports — and `node --test`
+  finds and runs test files as Node 22 does. `run()`, coverage and mock timers are not
+  implemented. A test file exits when its tests finish, as under `--test-force-exit`.
+- **On the in-realm runtime, programs share one global object.** A global one program sets is
+  visible to the next. The worker runtime (the default where shared memory is available) gives
+  each program its own.
 - **Python is source-built CPython/WASM.** Each program gets its own interpreter
   process worker. Pure-Python wheels install normally; native extensions must
   be linked or published for Emscripten. The bundled wheel index includes the

package/bin/sandboxedjs-build-wheels.mjs

@@ -0,0 +1,143 @@
+#!/usr/bin/env node
+/**
+ * A wheel index that builds what it does not have.
+ *
+ * A browser cannot compile a C extension and never will, so "build it from
+ * source" there can only mean "ask a machine that can". This is that machine,
+ * reduced to the smallest thing that works: it serves the wheel index over
+ * HTTP, and it builds a package on request using the same pipeline and the
+ * same ABI contract as everything else.
+ *
+ *   sandboxedjs-build-wheels [port=4180]
+ *
+ * Point a host at it:
+ *
+ *   configurePython({
+ *     wheelIndex: "http://localhost:4180",
+ *     buildFromSource: "http://localhost:4180/build",
+ *   });
+ *
+ * It binds loopback only. Building runs a package's own build system, which is
+ * arbitrary code execution by design, so exposing this to a network would be
+ * handing that to whoever can reach it.
+ */
+
+import { createServer } from "node:http";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), "../python-runtime");
+const wheels = join(root, "out/wheels");
+
+if (!existsSync(join(root, "scripts/build_extension.py"))) {
+  console.error("sandboxedjs-build-wheels: the build pipeline is only present in a checkout");
+  process.exit(1);
+}
+
+const port = Number(process.argv[2] ?? 4180);
+const building = new Map();
+const finished = new Map();
+
+const run = (args) => new Promise((done) => {
+  const child = spawn("python3", args, { cwd: root });
+  let stdout = "", stderr = "";
+  child.stdout.on("data", (c) => { stdout += String(c); });
+  child.stderr.on("data", (c) => { stderr += String(c); });
+  child.on("error", (e) => done({ code: -1, stdout, stderr: String(e) }));
+  child.on("close", (code) => done({ code: code ?? -1, stdout, stderr }));
+});
+
+async function build(requirement) {
+  const recipe = await run(["scripts/auto_recipe.py", requirement]);
+  if (recipe.code !== 0) {
+    let reported = {};
+    try { reported = JSON.parse(recipe.stdout.trim().split("\n").at(-1) ?? "{}"); } catch {}
+    return {
+      built: false,
+      classification: reported.classification ?? "blocked-toolchain",
+      reason: reported.reason ?? recipe.stderr.trim().slice(-400),
+    };
+  }
+  const path = recipe.stdout.trim().split("\n").at(-1);
+  const made = await run(["scripts/build_extension.py", path]);
+  if (made.code !== 0) {
+    return {
+      built: false, classification: "blocked-toolchain",
+      reason: `${made.stdout}\n${made.stderr}`.trim().split("\n").slice(-12).join("\n"),
+    };
+  }
+  const indexed = await run(["scripts/build_index.py"]);
+  if (indexed.code !== 0) {
+    return { built: false, classification: "blocked-toolchain", reason: indexed.stderr.slice(-300) };
+  }
+  return { built: true, classification: "supported-generic" };
+}
+
+const server = createServer(async (request, response) => {
+  const cors = {
+    "access-control-allow-origin": request.headers.origin ?? "*",
+    "access-control-allow-headers": "content-type",
+    "access-control-allow-methods": "GET,POST,OPTIONS",
+  };
+  if (request.method === "OPTIONS") { response.writeHead(204, cors).end(); return; }
+
+  const path = decodeURIComponent(new URL(request.url ?? "/", "http://localhost").pathname);
+
+  if (request.method === "POST" && path === "/build") {
+    const body = await new Promise((done) => {
+      let text = ""; request.on("data", (c) => { text += c; }); request.on("end", () => done(text));
+    });
+    let requirement = "";
+    try { requirement = String(JSON.parse(body || "{}").requirement ?? ""); } catch {}
+    /* Only a package name and optional pin. Anything else would be handed to a
+     * shell-adjacent build as an argument. */
+    if (!/^[A-Za-z0-9._-]+(==[A-Za-z0-9._+-]+)?$/.test(requirement)) {
+      response.writeHead(400, { ...cors, "content-type": "application/json" })
+        .end(JSON.stringify({ built: false, reason: "invalid requirement" }));
+      return;
+    }
+    /* Submit and poll, never one long request. A build takes minutes and an
+     * HTTP client -- especially one inside a sandboxed worker -- will drop a
+     * connection held open that long, which looks to the caller like the
+     * service being unreachable while it is in fact working. This is why the
+     * protocol in docs/python/build-on-miss.md has states rather than a single
+     * blocking call.
+     *
+     * One build per package: two identical requests are the same build. */
+    if (!building.has(requirement) && !finished.has(requirement)) {
+      console.log(`building ${requirement}`);
+      building.set(requirement, build(requirement).then((verdict) => {
+        console.log(`  ${requirement}: ${verdict.built ? "built" : verdict.classification}`);
+        finished.set(requirement, verdict);
+        building.delete(requirement);
+        return verdict;
+      }));
+    }
+    const done = finished.get(requirement);
+    const verdict = done
+      ? { ...done, state: done.built ? "published" : "failed" }
+      : { state: "building-wheel" };
+    response.writeHead(200, { ...cors, "content-type": "application/json" })
+      .end(JSON.stringify(verdict));
+    return;
+  }
+
+  const name = path === "/" ? "/index.json" : path;
+  const file = join(wheels, name);
+  if (!file.startsWith(wheels) || !existsSync(file)) {
+    response.writeHead(404, cors).end("not found");
+    return;
+  }
+  response.writeHead(200, {
+    ...cors,
+    "content-type": name.endsWith(".json") ? "application/json" : "application/octet-stream",
+  }).end(readFileSync(file));
+});
+
+server.listen(port, "127.0.0.1", () => {
+  console.log(`SandboxedJs wheel builder: http://127.0.0.1:${port}`);
+  console.log(`  index  http://127.0.0.1:${port}/index.json`);
+  console.log(`  build  POST http://127.0.0.1:${port}/build  {"requirement":"numpy"}`);
+});

package/dist/agent.d.cts

@@ -1,5 +1,5 @@
-import { C as Container } from './container-BwXE2J26.cjs';
-import './contracts-CVgctitO.cjs';
+import { C as Container } from './container-DAWPIkGG.cjs';
+import './contracts-C2_dTgk3.cjs';
 
 /**
  * Structural copies of the LangChain Deep Agents backend contract.

package/dist/agent.d.ts

@@ -1,5 +1,5 @@
-import { C as Container } from './container-WPf_A76-.js';
-import './contracts-CVgctitO.js';
+import { C as Container } from './container-CsLNpeyi.js';
+import './contracts-C2_dTgk3.js';
 
 /**
  * Structural copies of the LangChain Deep Agents backend contract.

package/dist/{container-BwXE2J26.d.cts → container-CsLNpeyi.d.ts}

@@ -1,4 +1,4 @@
-import { V as Vfs, C as Cred, f as RuntimePod, D as DirEntry, p as Stats } from './contracts-CVgctitO.cjs';
+import { V as Vfs, C as Cred, f as RuntimePod, O as OutboundPolicy, D as DirEntry, p as Stats } from './contracts-C2_dTgk3.js';
 
 /**
  * Byte streams for stdin/stdout/stderr, pipelines and redirections.
@@ -487,6 +487,8 @@ declare class NetworkStack {
         timeoutMs?: number;
         intervalMs?: number;
     }): Promise<boolean>;
+    /** The policy every way out of the container applies, not only the shell's. */
+    get policy(): OutboundPolicy;
     outboundAllowed(url: string): boolean;
     procNetDev(): string;
     procNetRoute(): string;
@@ -979,8 +981,13 @@ declare class Shell {
     captureSubshell(command: string): Promise<string>;
     /** `<(cmd)` — run the command now and hand back a path holding its output. */
     private makeProcessSubstitution;
-    /** A copy that shares nothing mutable with this shell. */
-    fork(): Shell;
+    /**
+     * A copy that shares nothing mutable with this shell.
+     *
+     * Subshells and pipeline stages run as part of this shell's process. A
+     * background job passes a process of its own, the way fork(2) gives one.
+     */
+    fork(proc?: Process): Shell;
     private currentIO;
     run(node: Node, io: ShellIO): Promise<number>;
     private runList;
@@ -1238,6 +1245,14 @@ interface PythonOptions {
      * no published wheel targets wasm32-emscripten.
      */
     wheelIndex?: WheelIndex | string | null;
+    /**
+     * Build a missing wheel instead of reporting it.
+     *
+     * `true` builds locally, which needs Node and the build pipeline. A URL asks
+     * a build service -- which is how a browser gets a wheel built, having no
+     * compiler of its own. See docs/python/build-on-miss.md.
+     */
+    buildFromSource?: boolean | string;
 }
 declare function configurePython(options?: PythonOptions): void;
 declare const isPythonAvailable: typeof isCPythonAvailable;
@@ -1535,6 +1550,14 @@ declare class Container {
     }): Session;
     /** The container-wide session used by `shell()` shorthand helpers. */
     get shell(): Session;
+    /**
+     * The streams one call collects into.
+     *
+     * They are detachable because a background job started by the command keeps
+     * writing after the call has returned. Nothing reads the result any more by
+     * then, so without `detach` a chatty server would grow the buffer for as long
+     * as it ran; afterwards its output still reaches the container-wide taps.
+     */
     private makeStdio;
     /** Send an HTTP request to a server running inside the container. */
     request(port: number, init?: {

package/dist/{container-WPf_A76-.d.ts → container-DAWPIkGG.d.cts}

@@ -1,4 +1,4 @@
-import { V as Vfs, C as Cred, f as RuntimePod, D as DirEntry, p as Stats } from './contracts-CVgctitO.js';
+import { V as Vfs, C as Cred, f as RuntimePod, O as OutboundPolicy, D as DirEntry, p as Stats } from './contracts-C2_dTgk3.cjs';
 
 /**
  * Byte streams for stdin/stdout/stderr, pipelines and redirections.
@@ -487,6 +487,8 @@ declare class NetworkStack {
         timeoutMs?: number;
         intervalMs?: number;
     }): Promise<boolean>;
+    /** The policy every way out of the container applies, not only the shell's. */
+    get policy(): OutboundPolicy;
     outboundAllowed(url: string): boolean;
     procNetDev(): string;
     procNetRoute(): string;
@@ -979,8 +981,13 @@ declare class Shell {
     captureSubshell(command: string): Promise<string>;
     /** `<(cmd)` — run the command now and hand back a path holding its output. */
     private makeProcessSubstitution;
-    /** A copy that shares nothing mutable with this shell. */
-    fork(): Shell;
+    /**
+     * A copy that shares nothing mutable with this shell.
+     *
+     * Subshells and pipeline stages run as part of this shell's process. A
+     * background job passes a process of its own, the way fork(2) gives one.
+     */
+    fork(proc?: Process): Shell;
     private currentIO;
     run(node: Node, io: ShellIO): Promise<number>;
     private runList;
@@ -1238,6 +1245,14 @@ interface PythonOptions {
      * no published wheel targets wasm32-emscripten.
      */
     wheelIndex?: WheelIndex | string | null;
+    /**
+     * Build a missing wheel instead of reporting it.
+     *
+     * `true` builds locally, which needs Node and the build pipeline. A URL asks
+     * a build service -- which is how a browser gets a wheel built, having no
+     * compiler of its own. See docs/python/build-on-miss.md.
+     */
+    buildFromSource?: boolean | string;
 }
 declare function configurePython(options?: PythonOptions): void;
 declare const isPythonAvailable: typeof isCPythonAvailable;
@@ -1535,6 +1550,14 @@ declare class Container {
     }): Session;
     /** The container-wide session used by `shell()` shorthand helpers. */
     get shell(): Session;
+    /**
+     * The streams one call collects into.
+     *
+     * They are detachable because a background job started by the command keeps
+     * writing after the call has returned. Nothing reads the result any more by
+     * then, so without `detach` a chatty server would grow the buffer for as long
+     * as it ran; afterwards its output still reaches the container-wide taps.
+     */
     private makeStdio;
     /** Send an HTTP request to a server running inside the container. */
     request(port: number, init?: {

package/dist/{contracts-CVgctitO.d.cts → contracts-C2_dTgk3.d.cts}

@@ -347,6 +347,27 @@ declare class VirtualTcpNetwork {
     closeAll(): void;
 }
 
+/**
+ * The outbound network policy, as plain functions every client consults.
+ *
+ * The container has several ways out — `curl` and `wget` in the shell, `http`,
+ * `https`, `fetch` and `WebSocket` in a Node program, sockets in Python — and
+ * they used to decide separately. Only the shell asked: a Node program's
+ * `fetch("https://…")` reached the internet from a container booted with
+ * outbound access off. One policy, applied at each exit, is what makes
+ * `network: { allowOutbound: false }` mean what it says.
+ *
+ * Loopback is not "outbound" at all. `127.0.0.1` inside the container is the
+ * container, so those requests are routed to its own servers and never handed
+ * to the host's network stack, whatever the policy allows.
+ */
+interface OutboundPolicy {
+    /** Whether requests may leave the container at all. */
+    allowOutbound: boolean;
+    /** When outbound is on, the hosts it may reach (subdomains included). `null` means any. */
+    allowedHosts: string[] | null;
+}
+
 /**
  * Clean-room contracts between SandboxedJS and its JavaScript runtime.
  *
@@ -479,9 +500,17 @@ interface RuntimePod {
         headers: Record<string, string>;
         body: Uint8Array;
     }) => Promise<RuntimeHttpResponse>): () => void;
+    /**
+     * Apply the container's outbound policy to programs this pod runs.
+     *
+     * Optional so a pod written elsewhere still satisfies the contract, but a pod
+     * without it cannot keep a program's own `fetch` inside the policy — only
+     * the shell's `curl` would honour it.
+     */
+    setNetworkPolicy?(policy: OutboundPolicy): void;
     snapshot(options?: Record<string, unknown>): unknown;
     restore(snapshot: unknown, options?: Record<string, unknown>): Promise<void>;
     teardown(): void;
 }
 
-export { type Cred as C, type DirEntry as D, type RuntimeVolume as R, type SpawnChild as S, Vfs as V, type WriteOptions as W, VirtualTcpNetwork as a, type RuntimeHttpResponse as b, type SyncSpawn as c, type VolumeStat as d, type VolumeStats as e, type RuntimePod as f, type RuntimePackageInstaller as g, type ChildSpawnConfig as h, type ChildHandle as i, type RuntimeProcess as j, type RuntimeSocketPeer as k, type RuntimeConnection as l, ROOT_CRED as m, type RuntimeProcessManager as n, type RuntimeProcessResult as o, Stats as p, type VirtualNode as q, type VirtualProvider as r, applyChmod as s, createChildProcessModule as t, formatMode as u, makeCred as v, octalMode as w, parseUmask as x };
+export { type Cred as C, type DirEntry as D, type OutboundPolicy as O, type RuntimeVolume as R, type SpawnChild as S, Vfs as V, type WriteOptions as W, VirtualTcpNetwork as a, type RuntimeHttpResponse as b, type SyncSpawn as c, type VolumeStat as d, type VolumeStats as e, type RuntimePod as f, type RuntimePackageInstaller as g, type ChildSpawnConfig as h, type ChildHandle as i, type RuntimeProcess as j, type RuntimeSocketPeer as k, type RuntimeConnection as l, ROOT_CRED as m, type RuntimeProcessManager as n, type RuntimeProcessResult as o, Stats as p, type VirtualNode as q, type VirtualProvider as r, applyChmod as s, createChildProcessModule as t, formatMode as u, makeCred as v, octalMode as w, parseUmask as x };

package/dist/{contracts-CVgctitO.d.ts → contracts-C2_dTgk3.d.ts}

@@ -347,6 +347,27 @@ declare class VirtualTcpNetwork {
     closeAll(): void;
 }
 
+/**
+ * The outbound network policy, as plain functions every client consults.
+ *
+ * The container has several ways out — `curl` and `wget` in the shell, `http`,
+ * `https`, `fetch` and `WebSocket` in a Node program, sockets in Python — and
+ * they used to decide separately. Only the shell asked: a Node program's
+ * `fetch("https://…")` reached the internet from a container booted with
+ * outbound access off. One policy, applied at each exit, is what makes
+ * `network: { allowOutbound: false }` mean what it says.
+ *
+ * Loopback is not "outbound" at all. `127.0.0.1` inside the container is the
+ * container, so those requests are routed to its own servers and never handed
+ * to the host's network stack, whatever the policy allows.
+ */
+interface OutboundPolicy {
+    /** Whether requests may leave the container at all. */
+    allowOutbound: boolean;
+    /** When outbound is on, the hosts it may reach (subdomains included). `null` means any. */
+    allowedHosts: string[] | null;
+}
+
 /**
  * Clean-room contracts between SandboxedJS and its JavaScript runtime.
  *
@@ -479,9 +500,17 @@ interface RuntimePod {
         headers: Record<string, string>;
         body: Uint8Array;
     }) => Promise<RuntimeHttpResponse>): () => void;
+    /**
+     * Apply the container's outbound policy to programs this pod runs.
+     *
+     * Optional so a pod written elsewhere still satisfies the contract, but a pod
+     * without it cannot keep a program's own `fetch` inside the policy — only
+     * the shell's `curl` would honour it.
+     */
+    setNetworkPolicy?(policy: OutboundPolicy): void;
     snapshot(options?: Record<string, unknown>): unknown;
     restore(snapshot: unknown, options?: Record<string, unknown>): Promise<void>;
     teardown(): void;
 }
 
-export { type Cred as C, type DirEntry as D, type RuntimeVolume as R, type SpawnChild as S, Vfs as V, type WriteOptions as W, VirtualTcpNetwork as a, type RuntimeHttpResponse as b, type SyncSpawn as c, type VolumeStat as d, type VolumeStats as e, type RuntimePod as f, type RuntimePackageInstaller as g, type ChildSpawnConfig as h, type ChildHandle as i, type RuntimeProcess as j, type RuntimeSocketPeer as k, type RuntimeConnection as l, ROOT_CRED as m, type RuntimeProcessManager as n, type RuntimeProcessResult as o, Stats as p, type VirtualNode as q, type VirtualProvider as r, applyChmod as s, createChildProcessModule as t, formatMode as u, makeCred as v, octalMode as w, parseUmask as x };
+export { type Cred as C, type DirEntry as D, type OutboundPolicy as O, type RuntimeVolume as R, type SpawnChild as S, Vfs as V, type WriteOptions as W, VirtualTcpNetwork as a, type RuntimeHttpResponse as b, type SyncSpawn as c, type VolumeStat as d, type VolumeStats as e, type RuntimePod as f, type RuntimePackageInstaller as g, type ChildSpawnConfig as h, type ChildHandle as i, type RuntimeProcess as j, type RuntimeSocketPeer as k, type RuntimeConnection as l, ROOT_CRED as m, type RuntimeProcessManager as n, type RuntimeProcessResult as o, Stats as p, type VirtualNode as q, type VirtualProvider as r, applyChmod as s, createChildProcessModule as t, formatMode as u, makeCred as v, octalMode as w, parseUmask as x };