render-workflows-dart 0.8.2

package/src/cli.js

@@ -0,0 +1,396 @@
+#!/usr/bin/env node
+// render-dart CLI: build, dev, init.
+
+const { spawn } = require('node:child_process');
+const { cp, mkdir, readFile, rename, writeFile } = require('node:fs/promises');
+const { existsSync, readdirSync, readFileSync } = require('node:fs');
+const path = require('node:path');
+
+const { version } = require('../package.json');
+const { resolveDart } = require('./toolchain/dart-sdk');
+const {
+  CHANNELS,
+  DEFAULT_DART_VERSION,
+  listVersions,
+  requestedVersion,
+} = require('./toolchain/dart-version');
+const { compile, findDartIoImports, isFresh, pubGet } = require('./toolchain/compile');
+const { nativeEntries, buildNative, writeGeneratedIgnores } = require('./toolchain/native');
+const { generate, ensureRuntimeFile } = require('./toolchain/generate');
+
+const log = (m) => console.log(`[render-dart] ${m}`);
+const fail = (m) => {
+  console.error(`[render-dart] ${m}`);
+  process.exit(1);
+};
+
+/** Reads render-dart settings from the project's package.json. */
+async function config(root) {
+  let pkg = {};
+  try {
+    pkg = JSON.parse(await readFile(path.join(root, 'package.json'), 'utf8'));
+  } catch {
+    // A project without package.json is fine; defaults apply.
+  }
+  const c = pkg.renderDart ?? {};
+  return {
+    entry: path.resolve(root, c.entry ?? 'tasks.dart'),
+    out: path.resolve(root, c.out ?? 'build/tasks.js'),
+    // Left undefaulted on purpose: the flag and the environment outrank this,
+    // and requestedVersion() cannot tell "unset" from "set to the default" if
+    // the default is applied here.
+    dartVersion: c.dartVersion,
+    optimize: c.optimize ?? 'O2',
+    sourceMaps: c.sourceMaps ?? false,
+    allowDartIo: c.allowDartIo ?? false,
+    // Directories whose dart:io use is legitimate — a local tool sitting
+    // beside the workflow, say. Narrower than allowDartIo, which switches the
+    // check off for task code too.
+    allowDartIoIn: c.allowDartIoIn ?? [],
+    native: c.native ?? [],
+  };
+}
+
+async function build(root, { force = false, dartVersion } = {}) {
+  const c = await config(root);
+
+  if (!existsSync(c.entry)) {
+    fail(
+      `No Dart entrypoint at ${path.relative(root, c.entry)}. ` +
+        `Create one, or set renderDart.entry in package.json.`,
+    );
+  }
+
+  let native;
+  try {
+    native = nativeEntries(root, c.native);
+  } catch (e) {
+    fail(e.message);
+  }
+  for (const entry of native) {
+    if (!existsSync(entry.entry)) {
+      fail(`renderDart.native lists ${entry.rel}, which does not exist.`);
+    }
+  }
+
+  // Native entries have their own content-hash cache, so the mtime check only
+  // decides whether dart2js needs to run again.
+  const jsFresh = !force && (await isFresh(root, c.out));
+  if (native.length === 0 && jsFresh) {
+    log('output is up to date, skipping compile');
+    return c.out;
+  }
+
+  if (!c.allowDartIo) {
+    // Declared native sources are compiled AOT, where dart:io works — as is
+    // native_task.dart, which is our own runtime for that side and is never
+    // reachable from the dart2js entrypoint.
+    const io = await findDartIoImports(root, [
+      ...native.map((n) => n.dir),
+      ...c.allowDartIoIn.map((d) => path.resolve(root, d)),
+      path.join(root, 'native_task.dart'),
+    ]);
+    if (io.length > 0) {
+      const where = io.map((h) => `  ${h.file}:${h.line}`).join('\n');
+      fail(
+        `dart:io is imported, but tasks are compiled with dart2js and dart:io ` +
+          `does not work there.\n\n${where}\n\n` +
+          `It compiles without error and then throws "Unsupported operation" ` +
+          `on the first run, so this would fail in production rather than here.\n\n` +
+          `Use package:http (which works — it goes through fetch) instead of ` +
+          `HttpClient, and dart:js_interop for anything else Node provides.\n` +
+          `Set "renderDart": { "allowDartIo": true } in package.json to skip ` +
+          `this check.`,
+      );
+    }
+  }
+
+  const asked = requestedVersion({
+    flag: dartVersion,
+    env: process.env.RENDER_DART_VERSION,
+    config: c.dartVersion,
+  });
+  if (asked.explicit) log(`Dart ${asked.version} requested by ${asked.from}`);
+
+  const { dart } = await resolveDart({
+    root,
+    version: asked.version,
+    explicit: asked.explicit,
+    log,
+  });
+
+  // Only needed when the project declares pub dependencies, but harmless
+  // otherwise and far cheaper than diagnosing a missing package_config.
+  let pubCache;
+  if (existsSync(path.join(root, 'pubspec.yaml'))) {
+    pubCache = pubGet(dart, root, log);
+  }
+
+
+  // Flag a render_dart.dart left behind by an older render-dart before
+  // anything can fail against a signature it does not have.
+  ensureRuntimeFile(root, 'render_dart.dart', log);
+
+  // Generation comes first: the stubs it writes are what tasks.dart imports,
+  // so dart2js must not run before they exist.
+  if (native.length > 0) {
+    try {
+      for (const entry of native) {
+        if (entry.mode === 'task') {
+          entry.main = generate({ dart, root, entry, pubCache, log });
+        }
+      }
+      writeGeneratedIgnores(root, native);
+      await buildNative({ dart, root, entries: native, pubCache, log });
+    } catch (e) {
+      fail(e.message);
+    }
+  }
+
+  if (jsFresh) {
+    log('JavaScript output is up to date, skipping compile');
+    return c.out;
+  }
+
+  await compile({
+    dart,
+    root,
+    entry: c.entry,
+    out: c.out,
+    optimize: c.optimize,
+    sourceMaps: c.sourceMaps,
+    pubCache,
+    log,
+  });
+  return c.out;
+}
+
+/** Builds, then hands off to the Render CLI's local task server. */
+async function dev(root, args) {
+  await build(root, { dartVersion: flagValue(args, '--dart-version') });
+
+  const startCommand = args.length > 0 ? args : ['node', 'index.js'];
+  log(`starting local task server: render workflows dev -- ${startCommand.join(' ')}`);
+
+  const child = spawn(
+    'render',
+    ['workflows', 'dev', '--', ...startCommand],
+    { cwd: root, stdio: 'inherit' },
+  );
+  child.on('error', (e) => {
+    if (e.code === 'ENOENT') {
+      fail(
+        'The Render CLI is not installed. Install it with `brew install render`, ' +
+          'or see https://render.com/docs/cli',
+      );
+    }
+    fail(e.message);
+  });
+  child.on('exit', (code) => process.exit(code ?? 0));
+}
+
+const EXAMPLES_DIR = path.join(__dirname, '..', 'examples');
+const RUNTIME_DIR = path.join(__dirname, '..', 'runtime');
+
+/** Template names, read from the directory rather than a hardcoded list. */
+function templates() {
+  return readdirSync(EXAMPLES_DIR, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name)
+    .sort();
+}
+
+/** Copies an example into a new directory as a starting point. */
+async function init(root, args) {
+  // Templates are the examples, so every one of them is provably runnable —
+  // there is no second copy to drift.
+  const template = args.find((a) => a.startsWith('--template='))?.split('=')[1] ??
+      (args.includes('--template') ? args[args.indexOf('--template') + 1] : null) ??
+      'default';
+
+  const available = templates();
+  if (!available.includes(template)) {
+    fail(`No template named "${template}". Available: ${available.join(', ')}`);
+  }
+
+  const positional = args.filter((a, i) =>
+      !a.startsWith('--') && args[i - 1] !== '--template');
+  const target = path.resolve(root, positional[0] ?? 'dart-workflow');
+  if (existsSync(target)) fail(`${target} already exists.`);
+
+  await mkdir(target, { recursive: true });
+  // An example is a working project, so its directory also holds build output
+  // and resolved dependencies. None of that belongs in a fresh scaffold, and
+  // copying a lockfile would pin someone to whatever was resolved here.
+  const skip = new Set([
+    'README.md',
+    'node_modules',
+    'build',
+    '.dart_tool',
+    'pubspec.lock',
+    'package-lock.json',
+  ]);
+
+  // The build writes a native/.gitignore naming what it generated. Reusing
+  // that list means a scaffold never carries a stale facade or stub, without
+  // this having to guess at their names.
+  const generatedList = path.join(EXAMPLES_DIR, template, 'native', '.gitignore');
+  if (existsSync(generatedList)) {
+    for (const line of readFileSync(generatedList, 'utf8').split('\n')) {
+      const name = line.trim();
+      if (name && !name.startsWith('#')) skip.add(name);
+    }
+  }
+
+  await cp(path.join(EXAMPLES_DIR, template), target, {
+    recursive: true,
+    filter: (src) => !skip.has(path.basename(src)),
+  });
+
+  // The Dart bridge files are not kept in the examples — `build` writes them
+  // when missing, which is also how they stay current on upgrade. A scaffold
+  // gets them up front so the project analyses before its first build.
+  for (const name of ['render_dart.dart', 'native_task.dart']) {
+    await cp(path.join(RUNTIME_DIR, name), path.join(target, name));
+  }
+
+  // Guidance for coding agents, in the one place they will reliably look.
+  // An agent helping in this project never opens node_modules, so nothing we
+  // ship inside the package reaches it.
+  for (const name of ['AGENTS.md', 'CLAUDE.md']) {
+    await cp(path.join(RUNTIME_DIR, name), path.join(target, name));
+  }
+
+  // npm strips .gitignore from published packages, so the template ships it
+  // as `gitignore` and it gets its real name back here.
+  const shipped = path.join(target, 'gitignore');
+  if (existsSync(shipped)) await rename(shipped, path.join(target, '.gitignore'));
+
+  // Name the project after its directory, and pin render-dart to whatever
+  // version is doing the scaffolding.
+  //
+  // The template used to carry a hardcoded range, which silently went stale:
+  // `^0.1.0` means `<0.2.0` for a 0.x package, so every project scaffolded
+  // after 0.2.0 quietly installed 0.1.1 and missed everything since. Deriving
+  // it here means it cannot drift again.
+  const pkgPath = path.join(target, 'package.json');
+  const pkg = JSON.parse(await readFile(pkgPath, 'utf8'));
+  pkg.name = path.basename(target);
+  // Drop the pre-0.8.2 name if the template still carries it, so a scaffold
+  // never ends up depending on both.
+  delete pkg.dependencies['render-dart'];
+  pkg.dependencies['render-workflows-dart'] = `^${version}`;
+  await writeFile(pkgPath, `${JSON.stringify(pkg, null, 2)}\n`);
+
+  log(`created ${path.relative(root, target) || target}`);
+  console.log(`
+Next:
+  cd ${path.relative(root, target) || target}
+  npm install
+  npx render-dart dev
+
+Then deploy with runtime "node", root directory "${path.basename(target)}",
+build command "npm install && npm run build", start command "npm start".
+`);
+}
+
+/** The value after `--name`, or after `--name=`. */
+function flagValue(args, name) {
+  const i = args.indexOf(name);
+  if (i !== -1 && args[i + 1] && !args[i + 1].startsWith('-')) return args[i + 1];
+  const inline = args.find((a) => a.startsWith(`${name}=`));
+  return inline ? inline.slice(name.length + 1) : undefined;
+}
+
+/** `render-dart dart` — what is available, and what this project will use. */
+async function dartInfo(root, args) {
+  if (args.includes('--list')) {
+    const channel = flagValue(args, '--channel') ?? 'stable';
+    const versions = await listVersions(channel);
+    const limit = Number(flagValue(args, '--limit') ?? 20);
+    console.log(`Dart ${channel} releases (${versions.length} total):\n`);
+    for (const v of versions.slice(0, limit)) console.log(`  ${v}`);
+    if (versions.length > limit) {
+      console.log(`\n  ... ${versions.length - limit} older, --limit N for more`);
+    }
+    console.log(
+      `\nPin one with --dart-version, RENDER_DART_VERSION, or\n` +
+        `"renderDart": { "dartVersion": "..." } in package.json.\n` +
+        `"latest" and the channel names ${CHANNELS.join(', ')} also work.`,
+    );
+    return;
+  }
+
+  // No flag: report what this project would actually use, which is the
+  // question people are usually asking.
+  const c = await config(root);
+  const asked = requestedVersion({
+    flag: flagValue(args, '--dart-version'),
+    env: process.env.RENDER_DART_VERSION,
+    config: c.dartVersion,
+  });
+  console.log(
+    `requested: ${asked.version}` +
+      (asked.explicit ? `  (from ${asked.from})` : '  (built-in default)'),
+  );
+  const resolved = await resolveDart({
+    root,
+    version: asked.version,
+    explicit: asked.explicit,
+    fetch: false,
+    log: () => {},
+  });
+  console.log(`resolved:  ${resolved.version ?? 'unknown'}  (${resolved.source})`);
+  console.log(`default:   ${DEFAULT_DART_VERSION}`);
+  console.log('\nrender-dart dart --list [--channel beta] [--limit N]');
+}
+
+async function main() {
+  const [command, ...args] = process.argv.slice(2);
+  const root = process.cwd();
+
+  switch (command) {
+    case 'build':
+      await build(root, {
+        force: args.includes('--force'),
+        dartVersion: flagValue(args, '--dart-version'),
+      });
+      break;
+    case 'dev':
+      await dev(root, args);
+      break;
+    case 'init':
+      await init(root, args);
+      break;
+    case 'dart':
+      await dartInfo(root, args);
+      break;
+    default:
+      console.log(`render-dart — write Render Workflows tasks in Dart
+
+Usage:
+  render-dart build [--force] [--dart-version <v>]
+                                Compile tasks.dart to build/tasks.js
+  render-dart dev [-- cmd...]   Build, then run the local task server
+  render-dart init [dir] [--template <name>]
+                                Scaffold a new Dart workflow project
+  render-dart dart [--list] [--channel <c>] [--limit <n>]
+                                Which Dart this project uses, or what exists
+
+Dart version, highest precedence first:
+  --dart-version <v>            also accepted by the dev command
+  RENDER_DART_VERSION=<v>
+  "renderDart": { "dartVersion": "<v>" } in package.json
+  A version may be exact, "latest", or a channel: stable, beta, dev.
+  An explicit one is used even if a different Dart is on PATH.
+
+Configure via "renderDart" in package.json:
+  entry, out, dartVersion, optimize, sourceMaps, allowDartIo, allowDartIoIn,
+  native
+`);
+      // Asking for help is not an error; an unknown command is.
+      process.exit(command && !['--help', '-h', 'help'].includes(command) ? 1 : 0);
+  }
+}
+
+main().catch((e) => fail(e.message));

package/src/native-worker.js

@@ -0,0 +1,196 @@
+// Keeping a native task executable alive between calls.
+//
+// Spawn-per-call costs a process start every time — small against Render's
+// per-run overhead, but real in a loop. A worker pays it once and then serves
+// requests over the same JSONL protocol, which is why the binary needs no
+// changes: its dispatch loop already reads until stdin closes.
+//
+// The trade-off is state. Top-level variables, caches and open handles persist
+// between calls, which is exactly what makes it fast and also means a leak
+// accumulates instead of being cleaned up by process exit. Opt-in per entry.
+const { spawn } = require('node:child_process');
+const path = require('node:path');
+
+/** binary path -> live worker */
+const workers = new Map();
+let nextId = 1;
+
+function startWorker(binary, idleTimeoutMs) {
+  const child = spawn(binary, [], {
+    cwd: process.cwd(),
+    stdio: ['pipe', 'pipe', 'pipe'],
+  });
+
+  const worker = {
+    child,
+    binary,
+    idleTimeoutMs,
+    pending: new Map(),
+    buffer: '',
+    stderr: '',
+    idleTimer: null,
+    dead: false,
+  };
+
+  child.stdout.setEncoding('utf8');
+  child.stdout.on('data', (chunk) => {
+    worker.buffer += chunk;
+    let nl;
+    while ((nl = worker.buffer.indexOf('\n')) !== -1) {
+      const line = worker.buffer.slice(0, nl);
+      worker.buffer = worker.buffer.slice(nl + 1);
+      routeLine(worker, line);
+    }
+  });
+
+  // Native code writing to stderr is diagnostics, not protocol. Keep it
+  // visible rather than swallowing it, and keep the tail for a crash report.
+  child.stderr.setEncoding('utf8');
+  child.stderr.on('data', (chunk) => {
+    worker.stderr = (worker.stderr + chunk).slice(-4096);
+    process.stderr.write(`[native ${path.basename(binary)}] ${chunk}`);
+  });
+
+  const die = (reason) => {
+    worker.dead = true;
+    clearTimeout(worker.idleTimer);
+    workers.delete(binary);
+    // A hung promise is the worst possible failure here — every waiting call
+    // gets a real error instead.
+    for (const { reject } of worker.pending.values()) {
+      reject(new Error(reason));
+    }
+    worker.pending.clear();
+  };
+
+  child.on('exit', (code, signal) => {
+    if (worker.pending.size === 0 && !worker.stderr) return void workers.delete(binary);
+    die(
+      `native worker ${path.basename(binary)} exited ` +
+        `${signal ? `on ${signal}` : `with code ${code}`}` +
+        `${worker.stderr.trim() ? `: ${worker.stderr.trim()}` : ''}`,
+    );
+  });
+  child.on('error', (e) => die(`native worker ${path.basename(binary)} failed to start: ${e.message}`));
+
+  return worker;
+}
+
+function routeLine(worker, line) {
+  if (!line.trim()) return;
+
+  let message;
+  try {
+    message = JSON.parse(line);
+  } catch {
+    // Not protocol — surface it rather than dropping it silently.
+    process.stderr.write(`[native ${path.basename(worker.binary)}] ${line}\n`);
+    return;
+  }
+
+  const call = worker.pending.get(message.id);
+  if (!call) return;
+
+  call.lines.push(line);
+  if ('$ok' in message || '$err' in message) {
+    worker.pending.delete(message.id);
+    call.resolve(call.lines);
+    armIdleTimer(worker);
+  }
+}
+
+function armIdleTimer(worker) {
+  clearTimeout(worker.idleTimer);
+  if (worker.idleTimeoutMs <= 0 || worker.pending.size > 0) return;
+
+  worker.idleTimer = setTimeout(() => {
+    if (worker.pending.size > 0) return;
+    workers.delete(worker.binary);
+    worker.child.stdin.end();
+    worker.child.kill();
+  }, worker.idleTimeoutMs);
+  // Never hold the process open just to wait for a reap.
+  worker.idleTimer.unref?.();
+}
+
+/**
+ * Sends one request line and resolves with every reply line for it.
+ *
+ * Returning raw lines keeps the Dart side identical for both process models —
+ * it parses `$log`, `$ok` and `$err` the same way whether they came from a
+ * worker or a one-shot spawn.
+ */
+function nativeCall(binary, requestLine, { idleTimeoutMs = 30000, timeoutMs = 0 } = {}) {
+  const resolved = path.resolve(process.cwd(), binary);
+
+  let worker = workers.get(resolved);
+  if (!worker || worker.dead || worker.child.exitCode !== null) {
+    worker = startWorker(resolved, idleTimeoutMs);
+    workers.set(resolved, worker);
+  }
+
+  // The id is assigned here, not in Dart: a worker multiplexes calls, so it
+  // has to be unique across everything in flight.
+  const id = nextId++;
+  const request = { ...JSON.parse(requestLine), id };
+
+  clearTimeout(worker.idleTimer);
+
+  return new Promise((resolve, reject) => {
+    // A worker handles requests one at a time, so a call that never returns
+    // would block every later one. The whole process goes, rather than leaving
+    // a worker wedged behind a hung handler.
+    let timer = null;
+    const settle = (fn) => (value) => {
+      if (timer) clearTimeout(timer);
+      fn(value);
+    };
+
+    worker.pending.set(id, { resolve: settle(resolve), reject: settle(reject), lines: [] });
+
+    if (timeoutMs > 0) {
+      timer = setTimeout(() => {
+        if (!worker.pending.has(id)) return;
+        worker.pending.delete(id);
+        worker.child.kill('SIGKILL');
+        reject(new Error(
+          `native worker ${path.basename(resolved)} exceeded ${timeoutMs} ms`,
+        ));
+      }, timeoutMs);
+      timer.unref?.();
+    }
+
+    worker.child.stdin.write(`${JSON.stringify(request)}\n`, (e) => {
+      if (!e) return;
+      const call = worker.pending.get(id);
+      worker.pending.delete(id);
+      call?.reject(new Error(
+        `could not write to native worker ${path.basename(resolved)}: ${e.message}`,
+      ));
+    });
+  });
+}
+
+/** Closes every worker's stdin, which its dispatch loop sees as EOF. */
+function shutdownWorkers() {
+  for (const worker of workers.values()) {
+    clearTimeout(worker.idleTimer);
+    try {
+      worker.child.stdin.end();
+    } catch {
+      // Already gone.
+    }
+  }
+  workers.clear();
+}
+
+function installNativeWorker() {
+  if (globalThis.__nativeCall) return;
+
+  globalThis.__nativeCall = (binary, requestLine, options) =>
+    nativeCall(binary, requestLine, options ?? {});
+
+  process.once('exit', shutdownWorkers);
+}
+
+module.exports = { installNativeWorker, nativeCall, shutdownWorkers, workers };