render-workflows-dart 0.8.2

package/src/node-bridge.js

@@ -0,0 +1,118 @@
+// Access to the Node platform that Dart cannot reach on its own.
+//
+// Two gaps, both structural rather than incidental:
+//
+//   `require` is module-scoped in CommonJS, and `globalThis.require` is
+//   undefined in both CommonJS and ESM -- verified, not assumed. A compiled
+//   Dart task therefore cannot load an npm package without a hoist.
+//
+//   `dart:io` compiles under dart2js and throws at runtime, so `Process` is
+//   unavailable. Shelling out has to go through Node.
+
+const { createRequire } = require('node:module');
+const path = require('node:path');
+
+/**
+ * Loads an npm package or Node built-in, by name.
+ *
+ * Dart cannot reach `require` on its own: in CommonJS it is module-scoped, and
+ * `globalThis.require` is undefined in both CommonJS and ESM. Hoisting it here
+ * is the only way a compiled Dart task can use the npm ecosystem.
+ *
+ *     final crypto = requireModule('node:crypto');
+ */
+function nodeRequire(id) {
+  return projectRequire()(id);
+}
+
+/**
+ * A `require` rooted at the project directory rather than at this file.
+ *
+ * Resolving from inside `node_modules/render-workflows-dart/src/` happens to reach the
+ * project's dependencies by walking up, but that is an accident of layout that
+ * pnpm and workspaces can break. Rooting at the working directory makes
+ * `requireModule('lodash')` mean what the task author expects: whatever their
+ * own package.json depends on.
+ */
+let cachedRequire;
+function projectRequire() {
+  cachedRequire ??= createRequire(path.join(process.cwd(), 'noop.js'));
+  return cachedRequire;
+}
+
+/**
+ * Runs a command to completion and captures its output.
+ *
+ * `dart:io` compiles under dart2js and then fails at runtime, so `Process` is
+ * unavailable to a task. This is the sanctioned way to shell out — to a CLI
+ * tool, or to a natively compiled Dart binary shipped alongside the workflow.
+ *
+ * Resolves with `{code, stdout, stderr}` even when the command exits non-zero:
+ * an exit code is a result, not an exception. It rejects only when the process
+ * could not be started at all.
+ */
+function runProcess(command, args, options) {
+  const { spawn } = require('node:child_process');
+  const opts = options ?? {};
+
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args ?? [], {
+      cwd: opts.cwd ?? process.cwd(),
+      env: opts.env ? { ...process.env, ...opts.env } : process.env,
+      shell: opts.shell ?? false,
+    });
+
+    let stdout = '';
+    let stderr = '';
+    let settled = false;
+
+    child.stdout?.setEncoding('utf8');
+    child.stderr?.setEncoding('utf8');
+    child.stdout?.on('data', (chunk) => (stdout += chunk));
+    child.stderr?.on('data', (chunk) => (stderr += chunk));
+
+    let timer;
+    if (opts.timeoutMs > 0) {
+      timer = setTimeout(() => {
+        // SIGKILL rather than SIGTERM: a task run is capped by Render's own
+        // timeout, and a process ignoring SIGTERM would burn the whole budget.
+        child.kill('SIGKILL');
+        if (!settled) {
+          settled = true;
+          reject(new Error(
+            `\`${command}\` exceeded its ${opts.timeoutMs}ms timeout`,
+          ));
+        }
+      }, opts.timeoutMs);
+    }
+
+    child.on('error', (e) => {
+      clearTimeout(timer);
+      if (!settled) {
+        settled = true;
+        reject(e);
+      }
+    });
+
+    child.on('close', (code, signal) => {
+      clearTimeout(timer);
+      if (settled) return;
+      settled = true;
+      resolve({ code: code ?? -1, signal: signal ?? null, stdout, stderr });
+    });
+
+    if (opts.stdin !== undefined && opts.stdin !== null) {
+      child.stdin?.end(opts.stdin);
+    } else {
+      child.stdin?.end();
+    }
+  });
+}
+
+/** Installs both globals. Safe to call more than once. */
+function installNodeBridge() {
+  if (globalThis.__require === undefined) globalThis.__require = nodeRequire;
+  if (globalThis.__run === undefined) globalThis.__run = runProcess;
+}
+
+module.exports = { installNodeBridge, nodeRequire, runProcess };

package/src/runtime.js

@@ -0,0 +1,108 @@
+// The Node-side runtime for Dart-authored Render Workflows tasks.
+//
+// Render runs a workflow's start command; that command loads this module,
+// which bridges dart2js output to the official @renderinc/sdk.
+
+// MUST run before @renderinc/sdk is required, and is why this module exists
+// rather than being inlined into user code.
+//
+// The SDK's task() schedules its own startTaskServer() via setImmediate as
+// soon as it sees RENDER_SDK_SOCKET_PATH. Since we also start the server
+// explicitly, leaving auto-start on produces TWO task servers and executes
+// every task body TWICE -- doubled side effects and doubled billing. Neither
+// `render workflows dev` nor Render in production sets this for us.
+process.env.RENDER_SDK_AUTO_START = 'false';
+
+const path = require('node:path');
+const { pathToFileURL } = require('node:url');
+const { task, startTaskServer } = require('@renderinc/sdk/workflows');
+const { installWebShims, ensureWasmRunGlobals } = require('./web-shims');
+const { installNodeBridge } = require('./node-bridge');
+const { installNativeWorker } = require('./native-worker');
+
+// Node lacks several APIs Dart packages assume: `self`, a `file:` scheme for
+// fetch, Dart's packages/<name>/ asset paths, and XMLHttpRequest.
+installWebShims();
+
+// Reaching npm packages and shelling out. Dart can do neither on its own:
+// `require` is module-scoped, and dart:io's Process fails under dart2js.
+installNodeBridge();
+installNativeWorker();
+
+/**
+ * Turns a project-relative path into a file: URL the patched fetch can read.
+ *
+ * Exposed to Dart so a task can point a package at a bundled asset, e.g.
+ * `initializeForge2D(wasmUri: Uri.parse(fileUri('web/box2d.wasm')))`.
+ */
+globalThis.__fileUri = (relativePath) =>
+  pathToFileURL(path.resolve(process.cwd(), relativePath)).href;
+
+/** Registered tasks, by name, as returned by the SDK's task(). */
+const wrapped = Object.create(null);
+
+/**
+ * Registers a Dart task. Called by the compiled Dart bundle, not by hand.
+ *
+ * Two adaptations happen here:
+ *  1. Varargs collapse into a single array, so the Dart side can expose
+ *     fixed-arity closures instead of guessing each task's parameter count.
+ *  2. The Dart result envelope is unwrapped. Dart must never throw across the
+ *     boundary: a converted Dart exception reaches Render as the opaque
+ *     "Dart exception thrown from converted Future...", with the real message
+ *     boxed out of reach. Task bodies return {$ok} or {$err} instead, and the
+ *     failure becomes a genuine Error here.
+ */
+globalThis.__registerTask = (name, fn, options) => {
+  wrapped[name] = task({ name, ...(options ?? {}) }, async (...args) => {
+    // Cached, so this is a no-op after the first task and costs nothing at all
+    // for projects that do not depend on wasm_run.
+    await ensureWasmRunGlobals();
+    const env = await fn(args);
+    if (env && env.$err !== undefined) throw new Error(env.$err);
+    return env ? env.$ok : undefined;
+  });
+};
+
+/**
+ * Invokes another task as a child run. Calling an SDK-wrapped function from
+ * inside an executing task is what makes Render spawn a subtask.
+ */
+globalThis.__callTask = (name, args) => {
+  const fn = wrapped[name];
+  if (!fn) {
+    throw new Error(
+      `Task '${name}' is not registered. Tasks must be registered at module ` +
+        `level, before the task server starts.`,
+    );
+  }
+  return fn(...args);
+};
+
+/**
+ * Starts the task server.
+ *
+ * The SDK reports a failure over /callback and then rethrows, which would
+ * otherwise surface as an unhandled rejection and a wall of compiled-JS
+ * stack. The run is already recorded failed by that point, so exit cleanly
+ * with the message.
+ */
+globalThis.__start = () =>
+  startTaskServer().catch((e) => {
+    console.error('[render-dart] task failed:', e && e.message ? e.message : e);
+    process.exit(1);
+  });
+
+/**
+ * Loads a compiled Dart bundle, which registers its tasks and starts serving.
+ *
+ * @param {string} bundlePath Path to dart2js output, relative to the caller.
+ */
+function runTasks(bundlePath = './build/tasks.js') {
+  const resolved = path.isAbsolute(bundlePath)
+    ? bundlePath
+    : path.resolve(process.cwd(), bundlePath);
+  require(resolved);
+}
+
+module.exports = { runTasks };

package/src/toolchain/compile.js

@@ -0,0 +1,153 @@
+// Compiling Dart to JavaScript, with a freshness check so the work is skipped
+// when nothing has changed.
+
+const { execFileSync, spawnSync } = require('node:child_process');
+const { mkdir, readdir, readFile, stat } = require('node:fs/promises');
+const path = require('node:path');
+
+const SKIP_DIRS = new Set(['build', 'node_modules', '.dart_tool', '.git']);
+
+/** Newest mtime across every .dart file under `dir`, recursively. */
+async function newestDartSource(dir) {
+  let newest = 0;
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch {
+    return 0;
+  }
+
+  for (const entry of entries) {
+    if (SKIP_DIRS.has(entry.name) || entry.name.startsWith('.')) continue;
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      newest = Math.max(newest, await newestDartSource(full));
+    } else if (entry.name.endsWith('.dart') || entry.name === 'pubspec.yaml') {
+      newest = Math.max(newest, (await stat(full)).mtimeMs);
+    }
+  }
+  return newest;
+}
+
+/** Whether `out` is newer than every Dart source under `root`. */
+async function isFresh(root, out) {
+  try {
+    const built = await stat(out);
+    return built.mtimeMs >= (await newestDartSource(root));
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Resolves pub dependencies, if the project declares any.
+ *
+ * dart2js needs a package resolution file before it can compile a project
+ * that imports anything from pub.dev, and Render's builder has no pub cache,
+ * so this has to run there too.
+ */
+function pubGet(dart, root, log) {
+  // Relocate the pub cache into node_modules for the same reason the SDK
+  // lives there: Render preserves node_modules between builds, but ~/.pub-cache
+  // sits in a fresh filesystem every time and would re-download every
+  // dependency on every deploy.
+  const pubCache = process.env.PUB_CACHE ?? path.join(root, 'node_modules', '.pub-cache');
+  const result = spawnSync(dart, ['pub', 'get'], {
+    cwd: root,
+    stdio: 'pipe',
+    env: { ...process.env, PUB_CACHE: pubCache },
+  });
+  if (result.status !== 0) {
+    const stderr = (result.stderr ?? '').toString().trim();
+    throw new Error(`dart pub get failed:\n${stderr}`);
+  }
+  log(`pub dependencies resolved (cache: ${path.relative(root, pubCache)})`);
+  return pubCache;
+}
+
+/**
+ * Compiles `entry` to `out` with dart2js.
+ *
+ * @param {object} opts
+ * @param {string} opts.dart      Path to the dart executable.
+ * @param {string} opts.root      Project root (holds pubspec.yaml).
+ * @param {string} opts.entry     Dart entrypoint.
+ * @param {string} opts.out       Output JavaScript file.
+ * @param {string} [opts.optimize] dart2js optimisation level, default O2.
+ * @param {boolean} [opts.sourceMaps] Emit a source map alongside the output.
+ * @param {(m: string) => void} opts.log
+ */
+async function compile({ dart, root, entry, out, optimize = 'O2', sourceMaps = false, pubCache, log }) {
+  await mkdir(path.dirname(out), { recursive: true });
+
+  const args = ['compile', 'js', `-${optimize}`, '-o', out];
+  // dart2js emits source maps by default; suppressing them keeps the bundle
+  // small, at the cost of production stack traces pointing into compiled JS.
+  if (!sourceMaps) args.push('--no-source-maps');
+  args.push(entry);
+
+  log(`compiling ${path.relative(root, entry)} -> ${path.relative(root, out)}`);
+  execFileSync(dart, args, {
+    cwd: root,
+    stdio: 'inherit',
+    env: pubCache ? { ...process.env, PUB_CACHE: pubCache } : process.env,
+  });
+
+  const { size } = await stat(out);
+  log(`done (${(size / 1024).toFixed(1)} KB)`);
+  return size;
+}
+
+/**
+ * Finds direct `dart:io` imports in the project's own Dart sources.
+ *
+ * This matters because dart2js **compiles dart:io without complaint** and then
+ * throws `Unsupported operation` at runtime — verified. A task using File,
+ * Process, Socket or HttpClient therefore deploys cleanly and fails on its
+ * first real run, which on Render means burning up to the task timeout before
+ * anyone finds out. Catching it at build time turns a production failure into
+ * a build error.
+ *
+ * Conditional imports (`if (dart.library.io)`) are legitimate and skipped.
+ * Only the project's own files are scanned, never pub dependencies.
+ */
+async function findDartIoImports(root, exemptDirs = []) {
+  const hits = [];
+  // Declared native sources are compiled AOT, where dart:io is the whole
+  // point. Their directories are exempt; everything else stays strict.
+  const exempt = exemptDirs.map((d) => path.resolve(d));
+  const isExempt = (full) =>
+    exempt.some((d) => full === d || full.startsWith(d + path.sep));
+
+  async function walk(dir) {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      if (SKIP_DIRS.has(entry.name) || entry.name.startsWith('.')) continue;
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (isExempt(full)) continue;
+        await walk(full);
+      } else if (entry.name.endsWith('.dart')) {
+        if (isExempt(full)) continue;
+        const source = await readFile(full, 'utf8');
+        source.split('\n').forEach((line, i) => {
+          const trimmed = line.trim();
+          if (!trimmed.startsWith('import') && !trimmed.startsWith('export')) return;
+          if (!/['"]dart:io['"]/.test(trimmed)) return;
+          if (trimmed.includes('if (')) return; // conditional import, fine
+          hits.push({ file: path.relative(root, full), line: i + 1 });
+        });
+      }
+    }
+  }
+
+  await walk(root);
+  return hits;
+}
+
+module.exports = { compile, isFresh, newestDartSource, pubGet, findDartIoImports };

package/src/toolchain/dart-sdk.js

@@ -0,0 +1,216 @@
+// Locating or fetching a Dart SDK.
+//
+// Deliberately free of Render specifics -- this is the piece worth extracting
+// into a general Dart/Node bridge if one is ever wanted.
+
+const { createHash } = require('node:crypto');
+const { execFileSync, spawnSync } = require('node:child_process');
+const { createWriteStream } = require('node:fs');
+const { mkdir, readFile, rm, stat, writeFile } = require('node:fs/promises');
+const { Readable } = require('node:stream');
+const { pipeline } = require('node:stream/promises');
+const path = require('node:path');
+
+const { resolveVersion } = require('./dart-version');
+
+const ARCHIVE_BASE =
+  'https://storage.googleapis.com/dart-archive/channels/stable/release';
+
+/**
+ * Where a vendored SDK is unpacked, relative to the project root.
+ *
+ * Inside node_modules on purpose. Render's build cache preserves node_modules
+ * between builds but not an arbitrary top-level directory -- measured: with
+ * the SDK at `<root>/.dart-sdk` it was re-downloaded on every build, 33s of a
+ * 52s build. Moved here, later builds reuse it and the build step drops to
+ * about a second. A dot-prefixed directory survives `npm install`.
+ */
+const VENDOR_DIR = path.join('node_modules', '.dart-sdk');
+
+async function exists(p) {
+  try {
+    await stat(p);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function onPath(bin) {
+  return spawnSync(bin, ['--version'], { stdio: 'ignore' }).status === 0;
+}
+
+/**
+ * The version of a `dart` executable, or null if it will not say.
+ *
+ * `dart --version` writes to stderr on some releases and stdout on others, so
+ * both are searched rather than assuming either.
+ */
+function versionOf(bin) {
+  const r = spawnSync(bin, ['--version'], { encoding: 'utf8' });
+  if (r.status !== 0) return null;
+  const m = `${r.stdout ?? ''}${r.stderr ?? ''}`.match(/Dart SDK version:\s*(\S+)/);
+  return m ? m[1] : null;
+}
+
+/** The version recorded beside a vendored SDK, if one was written. */
+async function vendoredVersion(root) {
+  try {
+    return (await readFile(path.join(root, VENDOR_DIR, 'VERSION'), 'utf8')).trim();
+  } catch {
+    return null;
+  }
+}
+
+/** The published checksum for an archive, or null if none is served. */
+async function publishedChecksum(url) {
+  const res = await fetch(`${url}.sha256sum`);
+  if (!res.ok) return null;
+  // The file is "<hex> *<filename>".
+  const [hex] = (await res.text()).trim().split(/\s+/);
+  return /^[0-9a-f]{64}$/.test(hex) ? hex : null;
+}
+
+/** The archive URL for a version on this platform. */
+function archiveUrl(version) {
+  const os = { linux: 'linux', darwin: 'macos', win32: 'windows' }[
+    process.platform
+  ];
+  const arch = { x64: 'x64', arm64: 'arm64' }[process.arch];
+  if (!os || !arch) {
+    throw new Error(
+      `No Dart SDK build for ${process.platform}/${process.arch}.`,
+    );
+  }
+  return `${ARCHIVE_BASE}/${version}/sdk/dartsdk-${os}-${arch}-release.zip`;
+}
+
+/** Downloads and unpacks a pinned SDK, returning the path to its `dart`. */
+async function fetchSdk({ root, version, log }) {
+  const dir = path.join(root, VENDOR_DIR);
+  const zip = path.join(root, 'node_modules', '.dart-sdk.zip');
+  const url = archiveUrl(version);
+
+  log(`fetching Dart ${version} (${url.split('/').pop()})`);
+  const [res, expected] = await Promise.all([
+    fetch(url),
+    publishedChecksum(url),
+  ]);
+  if (!res.ok) throw new Error(`Dart SDK download failed: ${res.status} ${url}`);
+
+  await mkdir(path.dirname(zip), { recursive: true });
+
+  // Hashed while it streams to disk rather than in a second pass: the bytes are
+  // already going through this process, so verifying costs no extra I/O and
+  // overlaps the download. Measured at 0.18s of CPU for a 228 MB archive,
+  // against roughly 30s to fetch it.
+  const hash = createHash('sha256');
+  await pipeline(
+    Readable.fromWeb(res.body),
+    async function* (source) {
+      for await (const chunk of source) {
+        hash.update(chunk);
+        yield chunk;
+      }
+    },
+    createWriteStream(zip),
+  );
+
+  if (expected) {
+    const actual = hash.digest('hex');
+    if (actual !== expected) {
+      await rm(zip, { force: true });
+      throw new Error(
+        `Dart SDK checksum mismatch for ${version}.\n` +
+          `  expected ${expected}\n  received ${actual}\n` +
+          'The archive was not unpacked. This is worth reporting rather than ' +
+          'retrying: the SDK is downloaded over the network and then executed.',
+      );
+    }
+    log('checksum verified');
+  } else {
+    // Older releases predate the published sums; say so rather than implying a
+    // check happened.
+    log(`no published checksum for ${version}; skipping verification`);
+  }
+
+  log('unpacking');
+  await rm(dir, { recursive: true, force: true });
+  await mkdir(dir, { recursive: true });
+  // The archive has a top-level `dart-sdk/`; unpack into a parent and point
+  // at the inner directory rather than trying to strip it.
+  execFileSync('unzip', ['-q', zip, '-d', dir], { stdio: 'inherit' });
+  await rm(zip, { force: true });
+
+  // Recorded so the next build can tell *which* SDK is cached. Without this the
+  // cache key is "does the directory exist", and changing the pin has no effect
+  // on any machine that has already built once — including every Render build
+  // after the first.
+  await writeFile(path.join(dir, 'VERSION'), `${version}\n`);
+
+  return path.join(dir, 'dart-sdk', 'bin', 'dart');
+}
+
+/**
+ * Finds a `dart` that satisfies the request, and says where it came from.
+ *
+ * The order still keeps each environment fast, but a pin now decides rather
+ * than merely suggesting:
+ *
+ *   1. a vendored SDK, if it is the version asked for
+ *   2. `dart` on PATH, if it is the version asked for — or if nothing was asked
+ *   3. download, unless `fetch` is false — see below
+ *
+ * The difference from before is the phrase "the version asked for". Previously
+ * both caches were consulted by existence alone, so a pinned version was
+ * honoured on a first Render build and silently ignored everywhere else: on a
+ * laptop PATH always won, and on later Render builds whatever had been vendored
+ * first won for ever. That is why setting `dartVersion` appeared to do nothing.
+ *
+ * A request that is not explicit — the built-in default — still defers to a
+ * local toolchain, because that default exists to give a first build something
+ * to fetch, not to override a Dart the developer installed deliberately.
+ */
+async function resolveDart({ root, version, explicit = false, fetch: mayFetch = true, log }) {
+  const wanted = await resolveVersion(version, { log });
+
+  const vendored = path.join(root, VENDOR_DIR, 'dart-sdk', 'bin', 'dart');
+  if (await exists(vendored)) {
+    const have = await vendoredVersion(root);
+    if (!explicit || have === wanted) {
+      log(`using Dart ${have ?? 'unknown'} (vendored)`);
+      return { dart: vendored, version: have, source: 'vendored' };
+    }
+    log(`vendored Dart is ${have ?? 'unknown'}, ${wanted} was asked for`);
+  }
+
+  if (onPath('dart')) {
+    const have = versionOf('dart');
+    if (!explicit || have === wanted) {
+      log(`using Dart ${have ?? 'unknown'} from PATH`);
+      return { dart: 'dart', version: have, source: 'path' };
+    }
+    log(`Dart on PATH is ${have ?? 'unknown'}, ${wanted} was asked for`);
+  }
+
+  if (!mayFetch) {
+    // Asking which Dart would be used must not install one. `render-dart dart`
+    // is a question, and a question that pulls 228 MB and unpacks 624 MB is a
+    // surprise nobody asked for.
+    return { dart: null, version: wanted, source: 'would download' };
+  }
+
+  const dart = await fetchSdk({ root, version: wanted, log });
+  log(`using Dart ${wanted} (downloaded)`);
+  return { dart, version: wanted, source: 'downloaded' };
+}
+
+module.exports = {
+  resolveDart,
+  fetchSdk,
+  archiveUrl,
+  versionOf,
+  vendoredVersion,
+  publishedChecksum,
+  VENDOR_DIR,
+};