render-workflows-dart 0.8.2

package/src/toolchain/dart-version.js

@@ -0,0 +1,113 @@
+// Deciding *which* Dart to use, separately from finding or fetching one.
+//
+// Kept apart from dart-sdk.js because the questions are different: this file
+// answers "what did the user ask for", that one answers "where is it". Mixing
+// them is how the pin came to be consulted only on the download path.
+
+const CHANNELS = ['stable', 'beta', 'dev'];
+
+/** The version this package installs when nobody has asked for one. */
+const DEFAULT_DART_VERSION = '3.13.1';
+
+const ARCHIVE = 'https://storage.googleapis.com/dart-archive/channels';
+
+/**
+ * What the user asked for, and how firmly.
+ *
+ * `explicit` is the important part. A pin someone typed has to be honoured even
+ * when a different Dart is already on PATH, or it is not a pin. The built-in
+ * default has no such claim: it exists so a first build on Render has something
+ * to fetch, and deferring to a local toolchain there is a courtesy, not a bug.
+ *
+ * Precedence, highest first: the command line, the environment, package.json.
+ * The flag is for trying something once, the environment for varying a build
+ * without a commit — Render's dashboard sets those — and package.json for the
+ * answer that should travel with the project.
+ */
+function requestedVersion({ flag, env, config }) {
+  if (flag) return { version: flag, explicit: true, from: '--dart-version' };
+  if (env) return { version: env, explicit: true, from: 'RENDER_DART_VERSION' };
+  if (config) return { version: config, explicit: true, from: 'package.json' };
+  return { version: DEFAULT_DART_VERSION, explicit: false, from: 'default' };
+}
+
+/** Whether a request names a moving target rather than one release. */
+function isAlias(version) {
+  return version === 'latest' || CHANNELS.includes(version);
+}
+
+/**
+ * Turns `latest` or a channel name into the version it currently means.
+ *
+ * An exact version resolves to itself without touching the network, so a
+ * pinned project keeps building when the archive is unreachable — which is the
+ * main practical argument for pinning one.
+ */
+async function resolveVersion(version, { log = () => {} } = {}) {
+  if (!isAlias(version)) return version;
+
+  const channel = version === 'latest' ? 'stable' : version;
+  const url = `${ARCHIVE}/${channel}/release/latest/VERSION`;
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(
+      `Could not resolve Dart "${version}": ${res.status} from ${url}`,
+    );
+  }
+  const { version: resolved } = await res.json();
+  log(`"${version}" is Dart ${resolved} today`);
+  return resolved;
+}
+
+/**
+ * Every version the archive offers on a channel, newest first.
+ *
+ * The bucket lists build-number directories alongside version ones — 28 against
+ * 176 on stable — so anything without a dot is dropped.
+ */
+async function listVersions(channel = 'stable') {
+  if (!CHANNELS.includes(channel)) {
+    throw new Error(
+      `Unknown channel "${channel}". Choose one of: ${CHANNELS.join(', ')}.`,
+    );
+  }
+  const url =
+    'https://storage.googleapis.com/storage/v1/b/dart-archive/o' +
+    `?delimiter=/&prefix=channels/${channel}/release/&fields=prefixes`;
+
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Could not list Dart versions: ${res.status}`);
+  const { prefixes = [] } = await res.json();
+
+  return prefixes
+    .map((p) => p.split('/').filter(Boolean).pop())
+    .filter((name) => name.includes('.'))
+    .sort(compareVersions)
+    .reverse();
+}
+
+/** Numeric where both are numeric, so 3.9.0 sorts below 3.10.0. */
+function compareVersions(a, b) {
+  const parts = (v) => v.split(/[.\-+]/).map((n) => (/^\d+$/.test(n) ? +n : n));
+  const [x, y] = [parts(a), parts(b)];
+  for (let i = 0; i < Math.max(x.length, y.length); i++) {
+    const [p, q] = [x[i], y[i]];
+    if (p === q) continue;
+    if (p === undefined) return -1;
+    if (q === undefined) return 1;
+    if (typeof p === typeof q) return p < q ? -1 : 1;
+    // A pre-release suffix sorts below the plain release it belongs to.
+    return typeof p === 'number' ? 1 : -1;
+  }
+  return 0;
+}
+
+module.exports = {
+  CHANNELS,
+  DEFAULT_DART_VERSION,
+  requestedVersion,
+  isAlias,
+  resolveVersion,
+  listVersions,
+  compareVersions,
+};

package/src/toolchain/generate.js

@@ -0,0 +1,112 @@
+// Driving the Dart generator that turns @nativeTask functions into a
+// dispatcher and typed stubs.
+//
+// The generator is a Dart program with its own pubspec, shipped inside this
+// package and resolved into the project's node_modules pub cache. Keeping it
+// self-contained means depending on package:analyzer never touches the user's
+// own pubspec.
+const { execFileSync, spawnSync } = require('node:child_process');
+const { existsSync, readFileSync, writeFileSync, mkdirSync } = require('node:fs');
+const path = require('node:path');
+
+const GENERATOR_DIR = path.join(__dirname, '..', '..', 'dart', 'generator');
+
+/** Resolves the generator's own dependencies, once. */
+function ensureGenerator(dart, pubCache, log) {
+  const marker = path.join(GENERATOR_DIR, '.dart_tool', 'package_config.json');
+  if (existsSync(marker)) return;
+
+  log('resolving generator dependencies (first native build)');
+  const result = spawnSync(dart, ['pub', 'get'], {
+    cwd: GENERATOR_DIR,
+    stdio: 'pipe',
+    env: { ...process.env, PUB_CACHE: pubCache ?? process.env.PUB_CACHE },
+  });
+  if (result.status !== 0) {
+    throw new Error(
+      `could not resolve the render-dart generator:\n${(result.stderr ?? '').toString().trim()}`,
+    );
+  }
+}
+
+/**
+ * Makes sure a runtime file the generated code imports is present.
+ *
+ * Written when missing. Never overwritten — an author may have edited it — but
+ * a difference is reported, because a stale copy alongside a newer render-dart
+ * is exactly how the `^0.1.0` template pin went unnoticed for three releases.
+ */
+function ensureRuntimeFile(root, name, log) {
+  const shipped = path.join(__dirname, '..', '..', 'runtime', name);
+  const local = path.join(root, name);
+
+  if (!existsSync(local)) {
+    writeFileSync(local, readFileSync(shipped));
+    log(`added ${name}`);
+    return;
+  }
+  if (readFileSync(local, 'utf8') !== readFileSync(shipped, 'utf8')) {
+    // Worth being loud about: the failure mode is a compile error against a
+    // signature that changed, which reads as a bug in the generated code
+    // rather than as a stale file.
+    log(
+      `note: ${name} differs from the copy shipped with this render-dart. ` +
+        `If the build fails on a signature it does not recognise, refresh it: ` +
+        `cp node_modules/render-workflows-dart/runtime/${name} ${name}`,
+    );
+  }
+}
+
+/**
+ * Generates the dispatcher and stubs for one native task entry.
+ *
+ * Returns the dispatcher path, which is what actually gets compiled — the
+ * author's file is imported by it rather than compiled directly.
+ */
+function generate({ dart, root, entry, pubCache, log }) {
+  ensureGenerator(dart, pubCache, log);
+  ensureRuntimeFile(root, 'native_task.dart', log);
+
+  const main = path.join(root, '.dart_tool', 'render_dart', `${entry.name}.main.dart`);
+  mkdirSync(path.dirname(main), { recursive: true });
+
+  // Only pass an override when package.json actually set one; otherwise the
+  // annotation decides.
+  const overrides = [];
+  if (entry.worker !== undefined) overrides.push('--worker', String(entry.worker));
+  if (entry.idleTimeoutMs !== undefined) overrides.push('--idle', String(entry.idleTimeoutMs));
+  if (entry.timeoutMs !== undefined) overrides.push('--timeout', String(entry.timeoutMs));
+
+  const result = spawnSync(
+    dart,
+    [
+      'run',
+      path.join('bin', 'generate.dart'),
+      '--project', root,
+      '--entry', entry.entry,
+      '--name', entry.name,
+      '--stub', entry.stub,
+      '--facade', entry.facade,
+      '--main', main,
+      ...overrides,
+    ],
+    {
+      cwd: GENERATOR_DIR,
+      stdio: 'pipe',
+      env: { ...process.env, PUB_CACHE: pubCache ?? process.env.PUB_CACHE },
+    },
+  );
+
+  if (result.status !== 0) {
+    throw new Error(
+      `native task ${entry.rel} could not be generated:\n` +
+        `${(result.stderr ?? '').toString().trim()}`,
+    );
+  }
+
+  const { methods } = JSON.parse((result.stdout ?? '{}').toString().trim() || '{}');
+  log(`native ${entry.name}: ${(methods ?? []).length} task(s) — ${(methods ?? []).join(', ')}`);
+  return main;
+}
+
+module.exports = { generate, ensureRuntimeFile, GENERATOR_DIR };

package/src/toolchain/index.js

@@ -0,0 +1,8 @@
+// The Dart -> JavaScript toolchain, free of Render specifics.
+//
+// Kept separate so it can be extracted into a general Dart/Node bridge if one
+// is ever wanted; nothing in here knows what a workflow is.
+module.exports = {
+  ...require('./dart-sdk'),
+  ...require('./compile'),
+};

package/src/toolchain/native.js

@@ -0,0 +1,217 @@
+// Compiling declared Dart files to native executables.
+//
+// Render's own container can do this: the SDK render-dart vendors is the full
+// SDK, carrying gen_snapshot and dartaotruntime, and the build host is already
+// linux/x64. So nothing is cross-compiled and no binary is committed — the
+// executable is produced from the source in the commit that deploys it.
+const { execFileSync } = require('node:child_process');
+const { createHash } = require('node:crypto');
+const { copyFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync, chmodSync } = require('node:fs');
+const { readdir } = require('node:fs/promises');
+const path = require('node:path');
+
+const CACHE_DIR = path.join('node_modules', '.native-cache');
+const OUT_DIR = path.join('build', 'native');
+const SKIP_DIRS = new Set(['build', 'node_modules', '.dart_tool', '.git']);
+
+/**
+ * Normalises `renderDart.native` into full entries.
+ *
+ * Accepts a bare path as shorthand for a wrapped native task, or an object for
+ * anything else:
+ *
+ *   "native": [
+ *     "native/image_tools.dart",
+ *     { "entry": "native/raw.dart", "mode": "exe" },
+ *     { "entry": "native/hot.dart", "worker": true, "idleTimeoutMs": 30000 }
+ *   ]
+ */
+function nativeEntries(root, declared) {
+  const seen = new Map();
+
+  return (declared ?? []).map((raw) => {
+    const spec = typeof raw === 'string' ? { entry: raw } : { ...raw };
+    if (!spec.entry) {
+      throw new Error('every renderDart.native entry needs an "entry" path');
+    }
+
+    const mode = spec.mode ?? 'task';
+    if (mode !== 'task' && mode !== 'exe') {
+      throw new Error(`unknown native mode "${mode}" for ${spec.entry} (expected "task" or "exe")`);
+    }
+
+    // Left undefined when package.json says nothing, so the annotation on the
+    // function decides. Only an explicit value here overrides it.
+    const worker = spec.worker;
+    // An "exe" owns its own main and has no dispatch loop to keep alive, so
+    // there is nothing for a worker to talk to.
+    if (worker === true && mode !== 'task') {
+      throw new Error(`"worker" needs mode "task", but ${spec.entry} is mode "exe"`);
+    }
+
+    const entry = path.resolve(root, spec.entry);
+    // `native/probe_impl.dart` is the implementation; the generated facade
+    // takes the plain name, so callers write `import 'native/probe.dart'` and
+    // never see a generated filename.
+    const base = path.basename(spec.entry, '.dart');
+    const name = base.endsWith('_impl') ? base.slice(0, -'_impl'.length) : base;
+
+    const dir = path.dirname(entry);
+    const facade = path.join(dir, `${name}.dart`);
+    if (facade === entry) {
+      throw new Error(
+        `${spec.entry} would be overwritten by its own generated facade. ` +
+          `Name the implementation ${name}_impl.dart, so the facade can take ` +
+          `${name}.dart and callers import the plain name.`,
+      );
+    }
+
+    if (seen.has(name)) {
+      throw new Error(
+        `two native entries are both named "${name}" (${seen.get(name)} and ${spec.entry}); ` +
+          `they would compile to the same executable`,
+      );
+    }
+    seen.set(name, spec.entry);
+
+    return {
+      name,
+      entry,
+      rel: spec.entry,
+      dir,
+      facade,
+      stub: path.join(dir, `${name}.stub.dart`),
+      mode,
+      worker,
+      idleTimeoutMs: spec.idleTimeoutMs,
+      timeoutMs: spec.timeoutMs,
+      out: path.join(root, OUT_DIR, name),
+    };
+  });
+}
+
+/** Every .dart file under `dir`, sorted, so a fingerprint is stable. */
+async function dartFilesUnder(dir) {
+  const found = [];
+  async function walk(d) {
+    let entries;
+    try {
+      entries = await readdir(d, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const e of entries) {
+      if (SKIP_DIRS.has(e.name) || e.name.startsWith('.')) continue;
+      const full = path.join(d, e.name);
+      if (e.isDirectory()) await walk(full);
+      else if (e.name.endsWith('.dart')) found.push(full);
+    }
+  }
+  await walk(dir);
+  return found.sort();
+}
+
+/**
+ * Content fingerprint of everything that feeds one native executable.
+ *
+ * Keyed on content, never mtime. Every Render deploy is a fresh git checkout,
+ * which stamps the current time on every file — so an mtime comparison can
+ * never hit, and the cache silently does nothing. That was measured against
+ * Render's build log before this was written.
+ */
+async function fingerprint(root, entry) {
+  const files = await dartFilesUnder(entry.dir);
+  const h = createHash('sha256');
+  h.update(entry.mode).update('\0').update(String(entry.worker)).update('\0');
+  for (const f of files) {
+    h.update(path.relative(root, f)).update('\0').update(readFileSync(f));
+  }
+  for (const meta of ['pubspec.yaml', 'pubspec.lock']) {
+    const p = path.join(root, meta);
+    if (existsSync(p)) h.update(meta).update('\0').update(readFileSync(p));
+  }
+  return h.digest('hex');
+}
+
+/**
+ * Compiles each entry, reusing a cached binary when its sources are unchanged.
+ *
+ * Output lands in node_modules/.native-cache first and is copied to
+ * build/native/. That is deliberate: Render's build cache preserves
+ * node_modules and nothing else, so a deploy touching only tasks.dart reuses
+ * the executable instead of paying for another AOT compile.
+ */
+async function buildNative({ dart, root, entries, pubCache, log }) {
+  const cacheDir = path.join(root, CACHE_DIR);
+  mkdirSync(cacheDir, { recursive: true });
+  mkdirSync(path.join(root, OUT_DIR), { recursive: true });
+
+  const env = pubCache ? { ...process.env, PUB_CACHE: pubCache } : process.env;
+
+  for (const entry of entries) {
+    // A "task" entry compiles the generated dispatcher, which imports the
+    // author's file; an "exe" entry compiles the file itself.
+    const source = entry.mode === 'task' ? entry.main : entry.entry;
+    const cached = path.join(cacheDir, entry.name);
+    const stamp = `${cached}.sha256`;
+    const want = await fingerprint(root, entry);
+
+    const hit =
+      existsSync(cached) &&
+      existsSync(stamp) &&
+      readFileSync(stamp, 'utf8').trim() === want;
+
+    if (hit) {
+      log(`native ${entry.name}: cache hit (${want.slice(0, 12)})`);
+    } else {
+      log(`native ${entry.name}: compiling ${path.relative(root, source)}`);
+      execFileSync(dart, ['compile', 'exe', source, '-o', cached], {
+        cwd: root,
+        stdio: 'inherit',
+        env,
+      });
+      writeFileSync(stamp, want);
+    }
+
+    copyFileSync(cached, entry.out);
+    chmodSync(entry.out, 0o755);
+    const { size } = statSync(entry.out);
+    log(`native ${entry.name}: ${path.relative(root, entry.out)} (${(size / 1e6).toFixed(1)} MB)`);
+  }
+}
+
+/**
+ * Writes a .gitignore listing what the build generated in each native dir.
+ *
+ * The facade takes a natural name like `probe.dart`, so without this it reads
+ * as hand-written source. Having the build maintain the list keeps it right as
+ * entries are added or renamed.
+ */
+function writeGeneratedIgnores(root, entries) {
+  const byDir = new Map();
+  for (const entry of entries) {
+    if (entry.mode !== 'task') continue;
+    const names = byDir.get(entry.dir) ?? [];
+    names.push(path.basename(entry.facade), path.basename(entry.stub));
+    byDir.set(entry.dir, names);
+  }
+
+  for (const [dir, names] of byDir) {
+    writeFileSync(
+      path.join(dir, '.gitignore'),
+      '# Written by `render-dart build`. These are generated from the\n' +
+        '# @nativeTask sources beside them and would only ever go stale in git.\n' +
+        `${names.sort().join('\n')}\n`,
+    );
+  }
+}
+
+module.exports = {
+  nativeEntries,
+  buildNative,
+  fingerprint,
+  dartFilesUnder,
+  writeGeneratedIgnores,
+  CACHE_DIR,
+  OUT_DIR,
+};