primitive-admin 1.1.0-alpha.45 → 1.1.0-alpha.47

package/dist/src/commands/workflows.js

@@ -1,4 +1,4 @@
-import { readFileSync, writeFileSync, statSync, existsSync, readdirSync, } from "fs";
+import { readFileSync, writeFileSync, statSync, readdirSync, } from "fs";
 import { basename } from "path";
 import * as path from "path";
 import { parseConfigToml, stringifyConfigToml } from "../lib/config-toml.js";
@@ -6,6 +6,7 @@ import { lookup as mimeLookup } from "mime-types";
 import { generateWorkflowTypes, } from "../lib/workflow-codegen/generator.js";
 import { ApiClient } from "../lib/api-client.js";
 import { resolveAppId } from "../lib/config.js";
+import { resolveCodegenSourceDir } from "../lib/codegen-shared/resolveCodegenSourceDir.js";
 import { validateWorkflowToml, formatWorkflowTomlErrors, } from "../lib/workflow-toml-validator.js";
 import { expandWorkflow } from "../lib/workflow-fragments.js";
 import { buildWorkflowPayloadFromToml } from "../lib/workflow-payload.js";
@@ -92,6 +93,75 @@ export function renderRunStatusStepResults(stepResults) {
     }
     return lines;
 }
+/**
+ * #1367 — annotate step-run rows with the inter-step gap: the wall-clock time
+ * between the previous step's `endedAt` and this step's `startedAt`. Sorts by
+ * `startedAt` when every row has a parseable one (tie-break `stepIndex`),
+ * otherwise by `stepIndex` for the whole list — the ordering mode must be
+ * chosen once for all rows, not per pair: a per-pair timestamp-vs-index
+ * fallback is not transitive when some rows lack a timestamp (A<B by index,
+ * B<C by index, C<A by time), and a non-transitive comparator leaves
+ * Array.prototype.sort free to return an arbitrary order, computing gaps
+ * against the wrong predecessor. The first step has no predecessor, so its
+ * `gapMs` is `null`. A gap is only meaningful when both adjacent timestamps
+ * parse and the gap is positive. When the immediate predecessor lacks a valid
+ * `endedAt`, the gap is left `null` rather than measured from an older,
+ * non-adjacent step — otherwise a later step could raise a false stall
+ * warning against a step it does not actually follow. Exported for
+ * testability.
+ */
+export function annotateStepGaps(steps) {
+    const rows = [...(steps || [])];
+    const allTimed = rows.every((r) => r?.startedAt && !Number.isNaN(new Date(r.startedAt).getTime()));
+    rows.sort((a, b) => {
+        if (allTimed) {
+            const sa = new Date(a.startedAt).getTime();
+            const sb = new Date(b.startedAt).getTime();
+            if (sa !== sb)
+                return sa - sb;
+        }
+        return (a?.stepIndex ?? 0) - (b?.stepIndex ?? 0);
+    });
+    let prevEnded = null;
+    for (const row of rows) {
+        const started = row?.startedAt ? new Date(row.startedAt).getTime() : NaN;
+        if (prevEnded != null && !Number.isNaN(started)) {
+            const gap = started - prevEnded;
+            row.gapMs = gap > 0 ? gap : 0;
+        }
+        else {
+            row.gapMs = null;
+        }
+        // Only genuinely adjacent steps yield a gap: if this step lacks a valid
+        // `endedAt`, clear the predecessor so the NEXT step's gap is left null
+        // instead of being measured from a non-adjacent earlier step.
+        const ended = row?.endedAt ? new Date(row.endedAt).getTime() : NaN;
+        prevEnded = Number.isNaN(ended) ? null : ended;
+    }
+    return rows;
+}
+/**
+ * #1367 — compute min / p50 / p95 / max over a set of latency samples (ms).
+ * Ignores non-finite values. Returns `null` when there are no valid samples.
+ * Percentiles use nearest-rank on the sorted samples. Exported for testability.
+ */
+export function summarizeLatencyDistribution(values) {
+    const nums = (values || []).filter((v) => typeof v === "number" && Number.isFinite(v));
+    if (nums.length === 0)
+        return null;
+    nums.sort((a, b) => a - b);
+    const pct = (p) => {
+        const rank = Math.ceil((p / 100) * nums.length);
+        return nums[Math.min(nums.length - 1, Math.max(0, rank - 1))];
+    };
+    return {
+        count: nums.length,
+        min: nums[0],
+        p50: pct(50),
+        p95: pct(95),
+        max: nums[nums.length - 1],
+    };
+}
 /**
  * Issue #687 (review feedback): render an unambiguous label for a workflow
  * config slot ("active config", "draft config", etc.) when the slot may be
@@ -837,6 +907,188 @@ Examples:
             process.exit(1);
         }
     });
+    // #1367 — hidden queue-latency measurement tool. Fires a batch of runs of a
+    // (trivial, no-op) workflow, polls until each has an executionStartedAt, and
+    // reports the observed queueDelayMs / createCallDurationMs distribution. This
+    // is the post-deploy verification vehicle: local miniflare does not exhibit
+    // real Cloudflare scheduling delay, so run it against an agents/alpha scratch
+    // app. Creates are paced (default 10/s) to stay clear of CF's ~100/s shared
+    // instance-creation limit; a create failure (e.g. 429) is recorded, not fatal.
+    workflows
+        .command("queue-latency-test", { hidden: true })
+        .description("(hidden) Fire N no-op runs and report the queue-delay distribution (post-deploy tool)")
+        .argument("<workflow-id>", "Workflow ID or key of a trivial no-op workflow")
+        .option("--app <app-id>", "App ID (uses current app if not specified)")
+        .option("--count <n>", "Number of runs to fire", "50")
+        .option("--rate <r>", "Max creates per second (<=10 recommended)", "10")
+        .option("--input <json>", "Root input as JSON")
+        .option("--timeout <sec>", "Seconds to wait for all runs to start", "120")
+        .option("--verbose", "Print a per-run row")
+        .option("--json", "Output as JSON")
+        .action(async (workflowId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        const count = Math.max(1, parseInt(options.count, 10) || 50);
+        const rate = Math.max(1, parseInt(options.rate, 10) || 10);
+        const timeoutSec = Math.max(1, parseInt(options.timeout, 10) || 120);
+        const intervalMs = Math.ceil(1000 / rate);
+        const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+        let rootInput;
+        if (options.input) {
+            try {
+                rootInput = JSON.parse(options.input);
+            }
+            catch {
+                error("Invalid JSON in --input");
+                process.exit(1);
+            }
+        }
+        const fired = [];
+        if (!options.json) {
+            info(`Firing ${count} run(s) of ${workflowId} at up to ${rate}/s (interval ${intervalMs}ms)...`);
+        }
+        for (let i = 0; i < count; i++) {
+            try {
+                const res = await client.previewWorkflow(resolvedAppId, workflowId, {
+                    rootInput,
+                });
+                fired.push({
+                    index: i,
+                    instanceId: res?.instanceId,
+                    createFailed: false,
+                    started: false,
+                });
+            }
+            catch (err) {
+                // A create failure (e.g. HTTP 500 from CF's shared-class rate limit) is
+                // a recorded datapoint, not a reason to abort the whole batch.
+                fired.push({
+                    index: i,
+                    createFailed: true,
+                    createError: err?.message ? String(err.message) : String(err),
+                    started: false,
+                });
+            }
+            if (i < count - 1)
+                await sleep(intervalMs);
+        }
+        // Poll the run list until every non-failed run has executionStartedAt set
+        // (queueDelayMs present) or we hit the timeout.
+        const byInstance = new Map();
+        for (const f of fired) {
+            if (f.instanceId)
+                byInstance.set(f.instanceId, f);
+        }
+        const deadline = Date.now() + timeoutSec * 1000;
+        const pending = () => fired.filter((f) => !f.createFailed && !f.started).length;
+        // The admin run-list endpoint caps each page at 200, so a single large
+        // request cannot cover a batch of more than 200 runs — the earliest ones
+        // would never be matched and would be falsely reported as not started.
+        // Page through with the cursor (runs come back newest-first, so our batch
+        // is at the front) until every pending run is matched or we run out of
+        // pages. Bound the scan so we never walk the whole run history.
+        const PAGE_LIMIT = 200;
+        const scanBudget = count + PAGE_LIMIT;
+        while (pending() > 0 && Date.now() < deadline) {
+            await sleep(1000);
+            try {
+                let cursor;
+                let scanned = 0;
+                do {
+                    const { items, cursor: nextCursor } = await client.listWorkflowRuns(resolvedAppId, workflowId, { limit: PAGE_LIMIT, cursor: cursor ?? undefined });
+                    for (const run of items || []) {
+                        const f = run?.instanceId ? byInstance.get(run.instanceId) : undefined;
+                        if (f && !f.started && run.executionStartedAt) {
+                            f.started = true;
+                            f.queueDelayMs =
+                                typeof run.queueDelayMs === "number" ? run.queueDelayMs : null;
+                            f.createCallDurationMs =
+                                typeof run.createCallDurationMs === "number"
+                                    ? run.createCallDurationMs
+                                    : null;
+                        }
+                    }
+                    scanned += items?.length || 0;
+                    cursor = nextCursor;
+                } while (pending() > 0 && cursor && scanned < scanBudget);
+            }
+            catch {
+                // Transient list error — keep polling until the deadline.
+            }
+        }
+        const createFailures = fired.filter((f) => f.createFailed).length;
+        const startedRuns = fired.filter((f) => f.started);
+        const queueDelays = startedRuns.map((f) => f.queueDelayMs);
+        const createDurations = startedRuns.map((f) => f.createCallDurationMs);
+        const queueDist = summarizeLatencyDistribution(queueDelays);
+        const createDist = summarizeLatencyDistribution(createDurations);
+        const notStarted = fired.filter((f) => !f.createFailed && !f.started).length;
+        if (options.json) {
+            json({
+                workflowId,
+                requested: count,
+                created: count - createFailures,
+                createFailures,
+                started: startedRuns.length,
+                notStartedWithinTimeout: notStarted,
+                queueDelayMs: queueDist,
+                createCallDurationMs: createDist,
+                runs: options.verbose
+                    ? fired.map((f) => ({
+                        index: f.index,
+                        instanceId: f.instanceId ?? null,
+                        createFailed: f.createFailed,
+                        createError: f.createError ?? null,
+                        started: f.started,
+                        queueDelayMs: f.queueDelayMs ?? null,
+                        createCallDurationMs: f.createCallDurationMs ?? null,
+                    }))
+                    : undefined,
+            });
+            return;
+        }
+        divider();
+        keyValue("Requested", count);
+        keyValue("Created", count - createFailures);
+        keyValue("Create failures", createFailures);
+        keyValue("Started", startedRuns.length);
+        keyValue("Not started (timeout)", notStarted);
+        divider();
+        if (queueDist) {
+            info("Queue delay (executionStartedAt - requested):");
+            keyValue("  min", formatDuration(queueDist.min));
+            keyValue("  p50", formatDuration(queueDist.p50));
+            keyValue("  p95", formatDuration(queueDist.p95));
+            keyValue("  max", formatDuration(queueDist.max));
+        }
+        else {
+            warn("No runs started — no queue-delay samples collected.");
+        }
+        if (createDist) {
+            divider();
+            info("Create-call duration:");
+            keyValue("  min", formatDuration(createDist.min));
+            keyValue("  p50", formatDuration(createDist.p50));
+            keyValue("  p95", formatDuration(createDist.p95));
+            keyValue("  max", formatDuration(createDist.max));
+        }
+        if (options.verbose) {
+            divider();
+            console.log(formatTable(fired.map((f) => ({
+                run: f.index,
+                instanceId: f.instanceId ?? "-",
+                status: f.createFailed ? "create-failed" : f.started ? "started" : "pending",
+                queueDelayMs: f.queueDelayMs,
+                createCallDurationMs: f.createCallDurationMs,
+            })), [
+                { header: "RUN", key: "run" },
+                { header: "INSTANCE", key: "instanceId", format: formatId },
+                { header: "STATE", key: "status" },
+                { header: "DELAY", key: "queueDelayMs", format: formatDuration },
+                { header: "CREATE", key: "createCallDurationMs", format: formatDuration },
+            ]));
+        }
+    });
     // Runs subcommand
     const runs = workflows.command("runs").description("Manage workflow runs");
     // List runs
@@ -868,6 +1120,8 @@ Examples:
                 { header: "RUN ID", key: "runId", format: formatId },
                 { header: "STATUS", key: "status", format: formatStatus },
                 { header: "STARTED", key: "startedAt", format: formatDate },
+                // #1367 — queue delay (requested → execution start).
+                { header: "DELAY", key: "queueDelayMs", format: formatDuration },
                 { header: "ENDED", key: "endedAt", format: formatDate },
                 { header: "PREVIEW", key: "isPreview", format: (v) => v ? "yes" : "" },
             ]));
@@ -897,7 +1151,13 @@ Examples:
             const run = result.run;
             keyValue("Run ID", run.runId);
             keyValue("Status", formatStatus(run.status));
-            keyValue("Started", formatDate(run.startedAt));
+            // #1367 — `startedAt` is the request time; `executionStartedAt` is when
+            // Cloudflare actually started running the instance. Queue delay is the
+            // gap. A still-queued run shows "-" for execution start / queue delay.
+            keyValue("Requested", formatDate(run.startedAt));
+            keyValue("Execution started", formatDate(run.executionStartedAt));
+            keyValue("Queue delay", formatDuration(run.queueDelayMs));
+            keyValue("Create call", formatDuration(run.createCallDurationMs));
             keyValue("Ended", formatDate(run.endedAt));
             keyValue("Preview", run.isPreview ? "yes" : "no");
             if (result.instanceStatus) {
@@ -947,7 +1207,12 @@ Examples:
                 info("No step runs found.");
                 return;
             }
-            console.log(formatTable(items, [
+            // #1367 — inter-step gap: idle time between one step ending and the
+            // next starting. Only surfaced above a 1s threshold so normal
+            // back-to-back steps stay quiet; a large gap flags a mid-run stall.
+            const STEP_GAP_THRESHOLD_MS = 1000;
+            const rows = annotateStepGaps(items);
+            console.log(formatTable(rows, [
                 { header: "STEP", key: "stepId" },
                 { header: "KIND", key: "stepKind" },
                 {
@@ -958,6 +1223,13 @@ Examples:
                             v === "skipped" ? chalk.gray(v) :
                                 v === "error_captured" ? chalk.yellow(v) : v,
                 },
+                {
+                    header: "GAP",
+                    key: "gapMs",
+                    format: (v) => typeof v === "number" && v >= STEP_GAP_THRESHOLD_MS
+                        ? chalk.yellow(formatDuration(v))
+                        : chalk.dim("-"),
+                },
                 { header: "DURATION", key: "durationMs", format: formatDuration },
                 {
                     header: "TOKENS",
@@ -2472,91 +2744,31 @@ Examples:
         .option("--json", "Output the result summary as JSON")
         .action(async (workflowKey, options) => {
         try {
-            // 1. Locate the workflows directory. Mirrors `databases codegen`'s
-            //    sync-dir resolution: honor --sync-dir, else search any
-            //    env/appId subtree under ./.primitive/sync that has a workflows/
-            //    directory. When --app is given, restrict the scan to that app's
-            //    subtree (the sync-tree layout is <env>/<appId>/), so a multi-app
-            //    tree generates only the requested app's workflows instead of
-            //    combining every app's and writing into the first one found.
-            const workflowDirs = [];
-            if (options.syncDir) {
-                const candidate = path.join(options.syncDir, "workflows");
-                if (existsSync(candidate)) {
-                    workflowDirs.push(candidate);
-                }
-            }
-            else {
-                const root = path.join(process.cwd(), ".primitive", "sync");
-                if (existsSync(root)) {
-                    for (const env of readdirSync(root)) {
-                        const envDir = path.join(root, env);
-                        try {
-                            for (const app of readdirSync(envDir)) {
-                                if (options.app && app !== options.app)
-                                    continue;
-                                const candidate = path.join(envDir, app, "workflows");
-                                if (existsSync(candidate)) {
-                                    workflowDirs.push(candidate);
-                                }
-                            }
-                        }
-                        catch {
-                            // skip non-dirs
-                        }
-                    }
-                }
-            }
-            if (workflowDirs.length === 0) {
-                error(options.app
-                    ? `No workflows/ directory found for app "${options.app}" under .primitive/sync/. Check the --app id, pass --sync-dir, or run \`primitive sync pull\` first.`
-                    : "No workflows/ directory found. Pass --sync-dir or run `primitive sync pull` first.");
-                process.exit(1);
-            }
-            // Codegen writes one <key>.generated.ts per workflow into a single
-            // output dir derived from the first source dir below. Generating from
-            // more than one discovered workflows/ directory (multiple apps, or the
-            // same app across multiple synced envs) would combine unrelated apps'
-            // workflows into the first one's output dir and write nothing for the
-            // rest. Require the caller to disambiguate rather than combining.
-            if (workflowDirs.length > 1) {
-                const appsFound = [
-                    ...new Set(workflowDirs.map((d) => path.basename(path.dirname(d)))),
-                ];
-                error(`Found workflows/ directories for more than one app/env:\n` +
-                    workflowDirs
-                        .map((d) => `  ${path.relative(process.cwd(), d)}`)
-                        .join("\n") +
-                    `\nCodegen writes into one output dir, so it must run against` +
-                    ` one app. ` +
-                    (options.app
-                        ? `App "${options.app}" resolves to multiple synced envs — pass --sync-dir to pick one.`
-                        : `Pass --app <app-id> (found: ${appsFound.join(", ")}) or` +
-                            ` --sync-dir to select one.`));
-                process.exit(1);
-            }
+            // 1. Resolve the single source workflows/ directory via the shared
+            //    active-environment resolver (issue #1510). Honors --sync-dir /
+            //    --app overrides, resolves the active env
+            //    (--env → PRIMITIVE_ENV → defaultEnvironment → single-env), and
+            //    preserves the legacy per-app scan fallback in bare-dir mode.
+            const workflowsSourceDir = resolveCodegenSourceDir({
+                subdir: "workflows",
+                options: { app: options.app, syncDir: options.syncDir },
+            });
             // 2. Collect the source .toml files (one per workflow), filtered to a
             //    single workflow when an argument is given. Match on the file stem;
             //    the generator resolves the real key from `[workflow].key`.
             const inputs = [];
-            const seenStems = new Set();
-            for (const dir of workflowDirs) {
-                for (const fileName of readdirSync(dir)) {
-                    if (!fileName.endsWith(".toml"))
-                        continue;
-                    const stem = fileName.slice(0, -".toml".length);
-                    if (workflowKey && stem !== workflowKey)
-                        continue;
-                    if (seenStems.has(stem))
-                        continue;
-                    seenStems.add(stem);
-                    const tomlPath = path.join(dir, fileName);
-                    inputs.push({
-                        fileStem: stem,
-                        tomlPath,
-                        tomlContent: readFileSync(tomlPath, "utf-8"),
-                    });
-                }
+            for (const fileName of readdirSync(workflowsSourceDir)) {
+                if (!fileName.endsWith(".toml"))
+                    continue;
+                const stem = fileName.slice(0, -".toml".length);
+                if (workflowKey && stem !== workflowKey)
+                    continue;
+                const tomlPath = path.join(workflowsSourceDir, fileName);
+                inputs.push({
+                    fileStem: stem,
+                    tomlPath,
+                    tomlContent: readFileSync(tomlPath, "utf-8"),
+                });
             }
             if (inputs.length === 0) {
                 error(workflowKey