stageflow 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +190 -13
  2. package/dist/agent/activity.d.ts +10 -1
  3. package/dist/agent/activity.js +37 -2
  4. package/dist/agent/activityObserver.d.ts +5 -1
  5. package/dist/agent/activityObserver.js +174 -11
  6. package/dist/agent/fakeAgent.js +4 -0
  7. package/dist/agent/piAdapter.d.ts +16 -0
  8. package/dist/agent/piAdapter.js +88 -21
  9. package/dist/agent/port.d.ts +2 -0
  10. package/dist/agent/providerAuth.js +12 -7
  11. package/dist/catalog/displayCatalogPath.d.ts +12 -0
  12. package/dist/catalog/displayCatalogPath.js +28 -0
  13. package/dist/cli/ciIdentity.d.ts +14 -0
  14. package/dist/cli/ciIdentity.js +52 -0
  15. package/dist/cli/initCommand.d.ts +9 -0
  16. package/dist/cli/initCommand.js +71 -0
  17. package/dist/cli/initTemplates.d.ts +3 -0
  18. package/dist/cli/initTemplates.js +19 -0
  19. package/dist/cli/operatorCatalog.d.ts +10 -0
  20. package/dist/cli/operatorCatalog.js +16 -0
  21. package/dist/cli/runCommand.d.ts +26 -0
  22. package/dist/cli/runCommand.js +209 -0
  23. package/dist/cli/runOutput.d.ts +19 -0
  24. package/dist/cli/runOutput.js +125 -0
  25. package/dist/cli/validateCommand.d.ts +1 -1
  26. package/dist/cli/validateCommand.js +20 -3
  27. package/dist/cli.d.ts +12 -1
  28. package/dist/cli.js +82 -66
  29. package/dist/config/browseCatalog.d.ts +45 -0
  30. package/dist/config/browseCatalog.js +170 -0
  31. package/dist/config/createPipeline.d.ts +12 -3
  32. package/dist/config/createPipeline.js +292 -100
  33. package/dist/config/createStage.d.ts +10 -4
  34. package/dist/config/createStage.js +44 -14
  35. package/dist/config/listConfig.d.ts +14 -19
  36. package/dist/config/listConfig.js +26 -191
  37. package/dist/config/listModelsFromManifest.d.ts +2 -0
  38. package/dist/config/listModelsFromManifest.js +4 -0
  39. package/dist/config/loadOutcome.d.ts +4 -0
  40. package/dist/config/loadPipeline.d.ts +1 -15
  41. package/dist/config/loadPipeline.js +84 -142
  42. package/dist/config/loadStage.d.ts +4 -0
  43. package/dist/config/loadStage.js +60 -32
  44. package/dist/config/loadStageflowManifest.d.ts +5 -0
  45. package/dist/config/loadStageflowManifest.js +157 -0
  46. package/dist/config/mergePipelineIncludes.d.ts +9 -0
  47. package/dist/config/mergePipelineIncludes.js +141 -0
  48. package/dist/config/normalizePipelineStageEntry.d.ts +16 -0
  49. package/dist/config/normalizePipelineStageEntry.js +186 -0
  50. package/dist/config/pipelineStageKeys.d.ts +4 -0
  51. package/dist/config/pipelineStageKeys.js +14 -0
  52. package/dist/config/resolveCatalogContext.d.ts +13 -0
  53. package/dist/config/resolveCatalogContext.js +13 -0
  54. package/dist/config/resolveForkEmitContext.d.ts +3 -0
  55. package/dist/config/resolveForkEmitContext.js +12 -0
  56. package/dist/config/resolvePipelineDag.d.ts +5 -1
  57. package/dist/config/resolvePipelineDag.js +54 -36
  58. package/dist/config/scanCatalogPaths.d.ts +4 -0
  59. package/dist/config/scanCatalogPaths.js +83 -0
  60. package/dist/config/validateCatalog.d.ts +22 -4
  61. package/dist/config/validateCatalog.js +169 -132
  62. package/dist/envelope/check.js +7 -0
  63. package/dist/envelope/forkChoice.d.ts +3 -0
  64. package/dist/envelope/forkChoice.js +47 -0
  65. package/dist/index.d.ts +4 -0
  66. package/dist/index.js +4 -0
  67. package/dist/mcp/projectRun.d.ts +3 -0
  68. package/dist/mcp/projectRun.js +7 -0
  69. package/dist/mcp/server.js +1 -1
  70. package/dist/mcp/tools.js +28 -11
  71. package/dist/project/findProjectRoot.d.ts +2 -0
  72. package/dist/project/findProjectRoot.js +51 -0
  73. package/dist/project/globalHome.d.ts +2 -0
  74. package/dist/project/globalHome.js +19 -0
  75. package/dist/project/resolveProjectContext.d.ts +9 -0
  76. package/dist/project/resolveProjectContext.js +38 -0
  77. package/dist/project/resolveStageflowContext.d.ts +11 -0
  78. package/dist/project/resolveStageflowContext.js +66 -0
  79. package/dist/runstore/normalizeCatalogPath.d.ts +1 -0
  80. package/dist/runstore/normalizeCatalogPath.js +4 -0
  81. package/dist/runstore/port.d.ts +15 -0
  82. package/dist/runstore/runProjection.js +11 -1
  83. package/dist/runstore/sqlite/SqliteRunStore.js +46 -4
  84. package/dist/runstore/sqlite/schema.d.ts +1 -1
  85. package/dist/runstore/sqlite/schema.js +7 -1
  86. package/dist/runtime/credentialBinding.d.ts +8 -6
  87. package/dist/runtime/credentialBinding.js +18 -24
  88. package/dist/runtime/pipelineRunner.d.ts +11 -0
  89. package/dist/runtime/pipelineRunner.js +25 -1
  90. package/dist/runtime/pipelineScheduler.d.ts +3 -0
  91. package/dist/runtime/pipelineScheduler.js +70 -14
  92. package/dist/runtime/reloadRunCatalog.d.ts +5 -0
  93. package/dist/runtime/reloadRunCatalog.js +44 -0
  94. package/dist/runtime/resumeReconstruct.d.ts +1 -0
  95. package/dist/runtime/resumeReconstruct.js +27 -10
  96. package/dist/runtime/runManager.d.ts +8 -0
  97. package/dist/runtime/runManager.js +49 -18
  98. package/dist/runtime/settingsFile.d.ts +12 -0
  99. package/dist/runtime/settingsFile.js +87 -6
  100. package/dist/runtime/stageAttemptBootstrap.js +3 -0
  101. package/dist/runtime/stageProcessLauncher.d.ts +1 -0
  102. package/dist/runtime/stageProcessLauncher.js +6 -0
  103. package/dist/runtime/stageRecovery.js +3 -1
  104. package/dist/runtime/stageRunner.d.ts +2 -0
  105. package/dist/runtime/stageRunner.js +9 -2
  106. package/dist/runtime/stageWorker.js +4 -6
  107. package/dist/runtime/stageWorkerProtocol.d.ts +1 -0
  108. package/dist/server/http.js +33 -9
  109. package/dist/tools/emitStageEnvelope.d.ts +3 -1
  110. package/dist/tools/emitStageEnvelope.js +11 -1
  111. package/dist/types/envelope.d.ts +1 -0
  112. package/dist/types/forkChoice.d.ts +8 -0
  113. package/dist/types/forkChoice.js +1 -0
  114. package/dist/types/pipeline.d.ts +51 -2
  115. package/dist/types/stageflowManifest.d.ts +23 -0
  116. package/dist/types/stageflowManifest.js +1 -0
  117. package/dist/ui/assets/index-CFSzDZje.js +118 -0
  118. package/dist/ui/assets/{index-DCsDopak.css → index-DefBlEvN.css} +1 -1
  119. package/dist/ui/index.html +3 -2
  120. package/dist/ui/stageflow-icon.svg +12 -0
  121. package/package.json +7 -5
  122. package/dist/agent/cursorExtension.d.ts +0 -12
  123. package/dist/agent/cursorExtension.js +0 -88
  124. package/dist/runstore/catalog.d.ts +0 -8
  125. package/dist/runstore/catalog.js +0 -16
  126. package/dist/runstore/disk/DiskRunStore.d.ts +0 -30
  127. package/dist/runstore/disk/DiskRunStore.js +0 -238
  128. package/dist/runstore/layout.d.ts +0 -22
  129. package/dist/runstore/layout.js +0 -58
  130. package/dist/runtime/hitlSeams.d.ts +0 -38
  131. package/dist/runtime/hitlSeams.js +0 -2
  132. package/dist/ui/assets/index-Cry0Tpfx.js +0 -118
@@ -1,10 +1,11 @@
1
1
  import { readFile } from "node:fs/promises";
2
2
  import path from "node:path";
3
- import { loadPipelineValidated } from "../config/loadPipeline.js";
3
+ import { loadPipelineValidated } from "../config/validateCatalog.js";
4
4
  import { loadTaskFromYaml } from "../config/loadTask.js";
5
5
  import { buildValidationResult } from "../config/validateCatalog.js";
6
6
  import { resolveAndValidateCheckout } from "./stageRoots.js";
7
7
  import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
8
+ import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
8
9
  import { runPipelineDag } from "./pipelineScheduler.js";
9
10
  import { readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
10
11
  import { StageProcessLauncher } from "./stageProcessLauncher.js";
@@ -41,12 +42,23 @@ async function preparePipeline(options) {
41
42
  const label = options.taskPath ?? "task.yaml";
42
43
  const task = loadTaskFromYaml(taskYaml, `task file ${label}`);
43
44
  const checkoutRoot = await resolveAndValidateCheckout(task, options.checkoutOverride, options.cwd);
45
+ const pipelinePath = normalizeCatalogPath(loaded.pipelinePath);
46
+ const taskPath = options.taskPath
47
+ ? normalizeCatalogPath(path.resolve(options.cwd, options.taskPath))
48
+ : undefined;
49
+ const projectRoot = normalizeCatalogPath(options.projectRoot ?? options.cwd);
44
50
  const run = await options.store.createRun({
45
51
  pipelineId: loaded.pipeline.id,
46
52
  taskYaml,
47
53
  taskId: task.id,
48
54
  checkoutRoot,
55
+ gitSha: options.gitSha,
56
+ ciPrUrl: options.ciPrUrl,
57
+ ciJobUrl: options.ciJobUrl,
49
58
  pipelineDag: buildPipelineDagSnapshotFromLoaded(loaded),
59
+ pipelinePath,
60
+ taskPath,
61
+ projectRoot,
50
62
  });
51
63
  const executionMode = readStageExecutionMode(process.env, options.executionMode);
52
64
  const stageProcessLauncher = resolveStageProcessLauncher(executionMode, options.stageProcessLauncher);
@@ -57,11 +69,13 @@ async function preparePipeline(options) {
57
69
  agent: options.agent,
58
70
  store: options.store,
59
71
  cwd: options.cwd,
72
+ projectRoot: options.projectRoot ?? options.cwd,
60
73
  checkoutRoot,
61
74
  hitl: options.hitl,
62
75
  executionMode,
63
76
  stageProcessLauncher,
64
77
  operatorCatalog: options.operatorCatalog,
78
+ skipGates: options.skipGates,
65
79
  };
66
80
  }
67
81
  export async function executeStages(prepared, options) {
@@ -82,6 +96,7 @@ export async function executeStages(prepared, options) {
82
96
  }
83
97
  export async function runPipeline(options) {
84
98
  const cwd = options.cwd ?? process.cwd();
99
+ const projectRoot = options.projectRoot ?? cwd;
85
100
  const prepared = await preparePipeline({
86
101
  agent: options.agent,
87
102
  store: options.store,
@@ -89,11 +104,13 @@ export async function runPipeline(options) {
89
104
  taskYaml: options.taskYaml,
90
105
  pipeline: options.pipeline,
91
106
  cwd,
107
+ projectRoot,
92
108
  checkoutOverride: options.checkoutOverride,
93
109
  hitl: options.hitl,
94
110
  executionMode: options.executionMode,
95
111
  stageProcessLauncher: options.stageProcessLauncher,
96
112
  operatorCatalog: options.operatorCatalog,
113
+ skipGates: options.skipGates,
97
114
  });
98
115
  return executeStages(prepared, {
99
116
  maxActiveStagesPerRun: options.maxActiveStagesPerRun,
@@ -104,6 +121,7 @@ export async function runPipeline(options) {
104
121
  /** Create the run immediately, then execute stages in the returned promise. */
105
122
  export async function startPipeline(options) {
106
123
  const cwd = options.cwd ?? process.cwd();
124
+ const projectRoot = options.projectRoot ?? cwd;
107
125
  const prepared = await preparePipeline({
108
126
  agent: options.agent,
109
127
  store: options.store,
@@ -111,11 +129,16 @@ export async function startPipeline(options) {
111
129
  taskYaml: options.taskYaml,
112
130
  pipeline: options.pipeline,
113
131
  cwd,
132
+ projectRoot,
114
133
  checkoutOverride: options.checkoutOverride,
134
+ gitSha: options.gitSha,
135
+ ciPrUrl: options.ciPrUrl,
136
+ ciJobUrl: options.ciJobUrl,
115
137
  hitl: options.hitl,
116
138
  executionMode: options.executionMode,
117
139
  stageProcessLauncher: options.stageProcessLauncher,
118
140
  operatorCatalog: options.operatorCatalog,
141
+ skipGates: options.skipGates,
119
142
  });
120
143
  const done = executeStages(prepared, {
121
144
  maxActiveStagesPerRun: options.maxActiveStagesPerRun,
@@ -125,6 +148,7 @@ export async function startPipeline(options) {
125
148
  await prepared.store.updateRunStatus(prepared.run.runId, "failed").catch(() => undefined);
126
149
  return {
127
150
  ok: false,
151
+ outcome: "failed",
128
152
  runDir: prepared.run.workspaceDir,
129
153
  runId: prepared.run.runId,
130
154
  reason: err instanceof Error ? err.message : String(err),
@@ -18,9 +18,11 @@ type SchedulerPreparedPipeline = {
18
18
  agent: AgentPort;
19
19
  store: RunStore;
20
20
  cwd: string;
21
+ projectRoot?: string;
21
22
  checkoutRoot?: string;
22
23
  hitl?: StageHitlController;
23
24
  operatorCatalog?: OperatorCatalog;
25
+ skipGates?: boolean;
24
26
  };
25
27
  export type { SchedulerPreparedPipeline };
26
28
  type RetryContext = {
@@ -85,6 +87,7 @@ export type RetryRunOptions = {
85
87
  onRetryRootTerminal?: OnRetryRootTerminal;
86
88
  };
87
89
  export declare function retryRun(options: RetryRunOptions): Promise<PipelineRunResult>;
90
+ export declare function applyForkSkipsFromEnvelopes(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>): void;
88
91
  export declare function applyRetryRootDelta(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>, retryRoots: Map<string, number>, stageId: string, attempt: number, clearSchedulingHalt?: () => void): void;
89
92
  export declare function hasWorkOutsideBranch(dag: ResolvedPipelineDag, branchRootId: string, states: Map<string, StageScheduleState>, ctx?: HasWorkOutsideBranchContext): boolean;
90
93
  export declare function runPipelineDag(options: RunPipelineDagOptions): Promise<PipelineRunResult>;
@@ -1,3 +1,4 @@
1
+ import { normalizeForkChoice } from "../envelope/forkChoice.js";
1
2
  import { collectDownstreamClosure, collectDownstreamStageIds, } from "./dagTraversal.js";
2
3
  import { buildCompletedEnvelopesFromRun, buildStageConfigById, } from "./envelopeRouting.js";
3
4
  import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
@@ -93,7 +94,8 @@ export async function resumeRun(options) {
93
94
  const hasActive = [...hydrated.states.values()].some((s) => s === "active");
94
95
  if (hasActive) {
95
96
  return {
96
- ok: true,
97
+ ok: false,
98
+ outcome: "waiting",
97
99
  runDir: prepared.run.workspaceDir,
98
100
  runId: prepared.run.runId,
99
101
  };
@@ -103,10 +105,14 @@ export async function resumeRun(options) {
103
105
  return state !== undefined && terminalState(state);
104
106
  });
105
107
  if (allTerminal) {
106
- const allSucceeded = dag.nodes.every((node) => hydrated.states.get(node.id) === "succeeded");
108
+ const allSucceeded = dag.nodes.every((node) => {
109
+ const s = hydrated.states.get(node.id);
110
+ return s === "succeeded" || s === "skipped";
111
+ });
107
112
  await prepared.store.updateRunStatus(prepared.run.runId, allSucceeded ? "succeeded" : "failed");
108
113
  return {
109
114
  ok: allSucceeded,
115
+ outcome: allSucceeded ? "succeeded" : "failed",
110
116
  runDir: prepared.run.workspaceDir,
111
117
  runId: prepared.run.runId,
112
118
  ...(allSucceeded
@@ -131,6 +137,33 @@ export async function retryRun(options) {
131
137
  onRetryRootTerminal: options.onRetryRootTerminal,
132
138
  });
133
139
  }
140
+ function applyForkChoiceToSchedule(dag, forkStageId, chosen, states, options) {
141
+ const skipPending = (stageId) => {
142
+ if (states.get(stageId) === "pending") {
143
+ states.set(stageId, "skipped");
144
+ options?.onSkip?.(stageId);
145
+ }
146
+ };
147
+ for (const childId of dag.childrenOf[forkStageId] ?? []) {
148
+ if (!chosen.has(childId)) {
149
+ skipPending(childId);
150
+ for (const desc of collectDownstreamStageIds(dag, childId)) {
151
+ skipPending(desc);
152
+ }
153
+ }
154
+ }
155
+ }
156
+ export function applyForkSkipsFromEnvelopes(dag, states, completedEnvelopes) {
157
+ for (const node of dag.nodes) {
158
+ if (!node.fork)
159
+ continue;
160
+ const envelope = completedEnvelopes.get(node.id);
161
+ if (!envelope)
162
+ continue;
163
+ const chosen = normalizeForkChoice(envelope.fork_choice, "stored");
164
+ applyForkChoiceToSchedule(dag, node.id, chosen, states);
165
+ }
166
+ }
134
167
  export function applyRetryRootDelta(dag, states, completedEnvelopes, retryRoots, stageId, attempt, clearSchedulingHalt) {
135
168
  retryRoots.set(stageId, attempt);
136
169
  const downstream = collectDownstreamClosure(dag, [stageId]);
@@ -204,7 +237,8 @@ function isRunnable(dag, stageId, states, schedulingHalted) {
204
237
  export async function runPipelineDag(options) {
205
238
  const { prepared, maxActiveStagesPerRun } = options;
206
239
  const executionMode = options.executionMode ?? "process";
207
- const { agent, task, loaded, run, store, checkoutRoot, hitl, cwd, } = prepared;
240
+ const { agent, task, loaded, run, store, checkoutRoot, hitl, cwd, projectRoot, } = prepared;
241
+ const factoryCwd = projectRoot ?? cwd ?? process.cwd();
208
242
  const dag = loaded.dag;
209
243
  const stageById = buildStageConfigById(loaded);
210
244
  let { states, completedEnvelopes, schedulingHalted, firstFailureReason } = options.initialSchedule ??
@@ -258,6 +292,7 @@ export async function runPipelineDag(options) {
258
292
  }
259
293
  }
260
294
  }
295
+ applyForkSkipsFromEnvelopes(dag, states, completedEnvelopes);
261
296
  let activeCount = 0;
262
297
  const inFlight = new Set();
263
298
  const notifyRetryRootTerminal = (stageId, outcome) => {
@@ -321,6 +356,13 @@ export async function runPipelineDag(options) {
321
356
  states.set(stageId, "succeeded");
322
357
  completedEnvelopes.set(stageId, envelope);
323
358
  notifyRetryRootTerminal(stageId, "succeeded");
359
+ const node = dag.nodes.find((n) => n.id === stageId);
360
+ if (node?.fork) {
361
+ const chosen = normalizeForkChoice(envelope.fork_choice, "stored");
362
+ applyForkChoiceToSchedule(dag, stageId, chosen, states, {
363
+ onSkip: (id) => notifyRetryRootTerminal(id, "skipped"),
364
+ });
365
+ }
324
366
  };
325
367
  const launchStage = async (stageId) => {
326
368
  const stage = stageById.get(stageId);
@@ -339,20 +381,20 @@ export async function runPipelineDag(options) {
339
381
  const launchResult = await launcher.launch({
340
382
  runId: run.runId,
341
383
  stageId,
342
- rootDir: cwd,
384
+ rootDir: factoryCwd,
343
385
  attempt,
344
386
  ...(prepared.operatorCatalog !== undefined
345
387
  ? { operatorCatalog: prepared.operatorCatalog }
346
388
  : {}),
389
+ ...(prepared.skipGates ? { skipGates: true } : {}),
347
390
  });
348
391
  if (launchResult.type === "succeeded") {
349
392
  try {
350
393
  const envelope = await store.readEnvelope(run.runId, stageId);
351
394
  onStageSuccess(stageId, envelope);
352
395
  }
353
- catch {
354
- states.set(stageId, "succeeded");
355
- notifyRetryRootTerminal(stageId, "succeeded");
396
+ catch (err) {
397
+ await onStageFailure(stageId, err instanceof Error ? err.message : String(err));
356
398
  }
357
399
  return;
358
400
  }
@@ -374,9 +416,10 @@ export async function runPipelineDag(options) {
374
416
  workspaceDir: run.workspaceDir,
375
417
  hitl,
376
418
  attemptCtx,
377
- factoryCwd: cwd,
419
+ factoryCwd,
378
420
  operatorCatalog: prepared.operatorCatalog,
379
421
  completedEnvelopes,
422
+ skipGates: prepared.skipGates,
380
423
  });
381
424
  if (isRunStageWaiting(result)) {
382
425
  await onStageFailure(stageId, WAIT_WITHOUT_WORKER_DISPATCH);
@@ -461,7 +504,12 @@ export async function runPipelineDag(options) {
461
504
  }
462
505
  const hasWaiting = [...states.values()].some((s) => s === "waiting");
463
506
  if (hasWaiting && !schedulingHalted) {
464
- return { ok: true, runDir: run.workspaceDir, runId: run.runId };
507
+ return {
508
+ ok: false,
509
+ outcome: "waiting",
510
+ runDir: run.workspaceDir,
511
+ runId: run.runId,
512
+ };
465
513
  }
466
514
  if (schedulingHalted) {
467
515
  markSkippedPending();
@@ -470,25 +518,33 @@ export async function runPipelineDag(options) {
470
518
  }
471
519
  return {
472
520
  ok: false,
521
+ outcome: "failed",
473
522
  runDir: run.workspaceDir,
474
523
  runId: run.runId,
475
524
  reason: firstFailureReason,
476
525
  };
477
526
  }
478
- const allSucceeded = dag.nodes.every((node) => states.get(node.id) === "succeeded");
527
+ const allSucceeded = dag.nodes.every((node) => {
528
+ const s = states.get(node.id);
529
+ return s === "succeeded" || s === "skipped";
530
+ });
479
531
  if (!allSucceeded) {
480
532
  if (retryContext === undefined) {
481
533
  await store.updateRunStatus(run.runId, "failed");
482
534
  }
483
535
  return {
484
536
  ok: false,
537
+ outcome: "failed",
485
538
  runDir: run.workspaceDir,
486
539
  runId: run.runId,
487
540
  reason: firstFailureReason ?? "pipeline incomplete",
488
541
  };
489
542
  }
490
- if (retryContext === undefined) {
491
- await store.updateRunStatus(run.runId, "succeeded");
492
- }
493
- return { ok: true, runDir: run.workspaceDir, runId: run.runId };
543
+ await store.updateRunStatus(run.runId, "succeeded");
544
+ return {
545
+ ok: true,
546
+ outcome: "succeeded",
547
+ runDir: run.workspaceDir,
548
+ runId: run.runId,
549
+ };
494
550
  }
@@ -0,0 +1,5 @@
1
+ import type { RunMeta } from "../runstore/port.js";
2
+ import type { LoadedPipeline } from "../types/pipeline.js";
3
+ import type { TaskFile } from "../types/task.js";
4
+ export declare function reloadPipelineForRun(meta: RunMeta): Promise<LoadedPipeline>;
5
+ export declare function reloadTaskForRun(meta: RunMeta): Promise<TaskFile>;
@@ -0,0 +1,44 @@
1
+ import { access } from "node:fs/promises";
2
+ import { loadPipeline } from "../config/loadPipeline.js";
3
+ import { loadTask } from "../config/loadTask.js";
4
+ import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
5
+ function missingLocator(field, runId) {
6
+ throw new Error(`Run ${runId} is missing ${field}; reload requires stored catalog locators.`);
7
+ }
8
+ function requireProjectRoot(meta) {
9
+ if (!meta.project_root) {
10
+ missingLocator("project_root", meta.run_id);
11
+ }
12
+ return normalizeCatalogPath(meta.project_root);
13
+ }
14
+ function requirePipelinePath(meta) {
15
+ if (!meta.pipeline_path) {
16
+ missingLocator("pipeline_path", meta.run_id);
17
+ }
18
+ return normalizeCatalogPath(meta.pipeline_path);
19
+ }
20
+ function requireTaskPath(meta) {
21
+ if (!meta.task_path) {
22
+ missingLocator("task_path", meta.run_id);
23
+ }
24
+ return normalizeCatalogPath(meta.task_path);
25
+ }
26
+ async function assertCatalogFileExists(filePath, label) {
27
+ try {
28
+ await access(filePath);
29
+ }
30
+ catch {
31
+ throw new Error(`${label} not found at stored path: ${filePath}`);
32
+ }
33
+ }
34
+ export async function reloadPipelineForRun(meta) {
35
+ const projectRoot = requireProjectRoot(meta);
36
+ const pipelinePath = requirePipelinePath(meta);
37
+ await assertCatalogFileExists(pipelinePath, "Pipeline");
38
+ return loadPipeline(pipelinePath, { cwd: projectRoot });
39
+ }
40
+ export async function reloadTaskForRun(meta) {
41
+ const taskPath = requireTaskPath(meta);
42
+ await assertCatalogFileExists(taskPath, "Task");
43
+ return loadTask(taskPath);
44
+ }
@@ -16,6 +16,7 @@ export type PreparedResumeContext = {
16
16
  executionMode: StageExecutionMode;
17
17
  stageProcessLauncher?: StageProcessLauncher;
18
18
  cwd: string;
19
+ factoryCwd?: string;
19
20
  maxActiveStagesPerRun: number;
20
21
  operatorCatalog?: OperatorCatalog;
21
22
  };
@@ -1,7 +1,9 @@
1
1
  import { existsSync } from "node:fs";
2
+ import { readFile } from "node:fs/promises";
2
3
  import { fakeHitlResumePath } from "../agent/fakeAgent.js";
3
- import { loadPipeline } from "../config/loadPipeline.js";
4
4
  import { loadTaskFromYaml } from "../config/loadTask.js";
5
+ import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
6
+ import { reloadPipelineForRun, reloadTaskForRun, } from "./reloadRunCatalog.js";
5
7
  import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
6
8
  import { resumeRun } from "./pipelineScheduler.js";
7
9
  import { buildStageRoots, withResolvedAuthPath } from "./stageRoots.js";
@@ -10,14 +12,27 @@ import { attemptContext } from "./stageAttemptContext.js";
10
12
  import { openStageAttempt, } from "./stageAttemptBootstrap.js";
11
13
  export async function loadRunContext(store, runId, cwd) {
12
14
  const meta = await store.readRunMeta(runId);
13
- const taskYaml = await store.readTaskYaml(runId);
14
- const task = loadTaskFromYaml(taskYaml, `run ${runId} task`);
15
- const loaded = await loadPipeline(meta.pipeline_id, { cwd });
15
+ const reloadMeta = {
16
+ ...meta,
17
+ project_root: meta.project_root ?? normalizeCatalogPath(cwd),
18
+ };
19
+ const loaded = await reloadPipelineForRun(reloadMeta);
20
+ let taskYaml;
21
+ let task;
22
+ if (meta.task_path) {
23
+ task = await reloadTaskForRun(meta);
24
+ taskYaml = await readFile(normalizeCatalogPath(meta.task_path), "utf8");
25
+ }
26
+ else {
27
+ taskYaml = await store.readTaskYaml(runId);
28
+ task = loadTaskFromYaml(taskYaml, `run ${runId} task`);
29
+ }
16
30
  const workspaceDir = store.getWorkspaceDir(runId);
17
31
  return { meta, taskYaml, task, loaded, workspaceDir };
18
32
  }
19
33
  export async function reconstructAndContinue(ctx) {
20
34
  const { runId, stageId, opaqueAnswer, agent, store, cwd } = ctx;
35
+ const factoryCwd = ctx.factoryCwd ?? cwd;
21
36
  try {
22
37
  const latestExecution = await store.getLatestStageExecution(runId, stageId);
23
38
  const attemptCtx = attemptContext(latestExecution?.attempt ?? 1);
@@ -36,7 +51,7 @@ export async function reconstructAndContinue(ctx) {
36
51
  const stage = loaded.stages[stageIndex];
37
52
  const workspaceDir = store.getWorkspaceDir(runId);
38
53
  const checkoutRoot = meta.checkout_root;
39
- const roots = withResolvedAuthPath(buildStageRoots(workspaceDir, stageId, checkoutRoot, attemptCtx), cwd);
54
+ const roots = withResolvedAuthPath(buildStageRoots(workspaceDir, stageId, checkoutRoot, attemptCtx), factoryCwd);
40
55
  const fakeResume = fakeHitlResumePath(roots, stageId);
41
56
  const piSession = attemptCtx.sessionPath(workspaceDir, stageId);
42
57
  if (!existsSync(fakeResume) && !existsSync(piSession)) {
@@ -57,7 +72,7 @@ export async function reconstructAndContinue(ctx) {
57
72
  dag: loaded.dag,
58
73
  checkoutRoot,
59
74
  workspaceDir,
60
- factoryCwd: cwd,
75
+ factoryCwd,
61
76
  attemptCtx,
62
77
  operatorCatalog: ctx.operatorCatalog,
63
78
  onActivity: (event) => {
@@ -86,7 +101,7 @@ export async function reconstructAndContinue(ctx) {
86
101
  skipStarted: true,
87
102
  existingHandle: opened.handle,
88
103
  attemptCtx,
89
- factoryCwd: cwd,
104
+ factoryCwd,
90
105
  operatorCatalog: ctx.operatorCatalog,
91
106
  });
92
107
  if (isRunStageWaiting(result)) {
@@ -108,6 +123,7 @@ export async function reconstructAndContinue(ctx) {
108
123
  agent,
109
124
  store,
110
125
  cwd,
126
+ projectRoot: factoryCwd,
111
127
  checkoutRoot,
112
128
  hitl: ctx.hitl,
113
129
  operatorCatalog: ctx.operatorCatalog,
@@ -118,9 +134,10 @@ export async function reconstructAndContinue(ctx) {
118
134
  executionMode: ctx.executionMode,
119
135
  stageProcessLauncher: ctx.stageProcessLauncher,
120
136
  });
121
- return rest.ok
122
- ? { ok: true }
123
- : { ok: false, reason: rest.reason };
137
+ if (rest.outcome === "failed") {
138
+ return { ok: false, reason: rest.reason };
139
+ }
140
+ return { ok: true };
124
141
  }
125
142
  catch (err) {
126
143
  const reason = err instanceof Error
@@ -61,10 +61,14 @@ export declare class RunManager {
61
61
  private readonly retryCoordinator;
62
62
  private readonly retryTracking;
63
63
  private readonly cwd;
64
+ private readonly projectRoot;
65
+ private readonly isGitProject;
64
66
  constructor(options: {
65
67
  agent: AgentPort;
66
68
  store: RunStore;
67
69
  cwd?: string;
70
+ projectRoot?: string;
71
+ isGitProject?: boolean;
68
72
  operatorCatalog?: OperatorCatalog;
69
73
  seams?: HitlSeams;
70
74
  maxConcurrent?: number;
@@ -100,6 +104,10 @@ export declare class RunManager {
100
104
  pipeline: string;
101
105
  task?: string | TaskFile;
102
106
  checkoutOverride?: string;
107
+ skipGates?: boolean;
108
+ gitSha?: string;
109
+ ciPrUrl?: string;
110
+ ciJobUrl?: string;
103
111
  }): Promise<StartRunResult>;
104
112
  rerun(runId: string): Promise<StartRunResult>;
105
113
  retryStage(runId: string, stageId: string): Promise<RetryStageResult>;
@@ -1,14 +1,15 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { readFile, realpath } from "node:fs/promises";
3
3
  import path from "node:path";
4
- import { loadPipeline } from "../config/loadPipeline.js";
4
+ import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
5
+ import { loadRunContext } from "./resumeReconstruct.js";
5
6
  import { loadTaskFromYaml } from "../config/loadTask.js";
6
7
  import { PipelineValidationError, startPipeline, } from "./pipelineRunner.js";
7
8
  import { resumeRun } from "./pipelineScheduler.js";
8
9
  import { RunRetryCoordinator, } from "./runRetryCoordinator.js";
9
10
  import { readMaxActiveStageProcesses, readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
10
11
  import { StageProcessLauncher } from "./stageProcessLauncher.js";
11
- import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, readMaxConcurrentFromFile, writeMaxConcurrentToFile, } from "./settingsFile.js";
12
+ import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, projectSettingsContext, readMaxConcurrentFromContext, writeMaxConcurrentToContext, } from "./settingsFile.js";
12
13
  import { StageHitlController, waitKey, } from "./stageHitl.js";
13
14
  import { resolveAndValidateCheckout, } from "./stageRoots.js";
14
15
  import { orchestrateAnswerResume } from "./answerResume.js";
@@ -79,12 +80,17 @@ export class RunManager {
79
80
  },
80
81
  };
81
82
  cwd;
83
+ projectRoot;
84
+ isGitProject;
82
85
  constructor(options) {
83
86
  this.options = options;
84
87
  this.cwd = options.cwd ?? process.cwd();
88
+ this.projectRoot = options.projectRoot ?? this.cwd;
89
+ this.isGitProject = options.isGitProject ?? false;
90
+ const settingsCtx = projectSettingsContext(this.cwd, this.projectRoot, this.isGitProject);
85
91
  this.maxConcurrent =
86
92
  options.maxConcurrent ??
87
- readMaxConcurrentFromFile(this.cwd) ??
93
+ readMaxConcurrentFromContext(settingsCtx) ??
88
94
  parseMaxConcurrent(process.env.STAGEFLOW_MAX_CONCURRENT_RUNS);
89
95
  this.maxActiveStagesPerRun = readMaxActiveStagesPerRun(process.env, options.maxActiveStagesPerRun);
90
96
  this.executionMode = readStageExecutionMode(process.env, options.executionMode);
@@ -106,7 +112,7 @@ export class RunManager {
106
112
  throw new Error(INVALID_SLOT_COUNT_MESSAGE);
107
113
  }
108
114
  this.maxConcurrent = parsed;
109
- writeMaxConcurrentToFile(this.cwd, parsed);
115
+ writeMaxConcurrentToContext(projectSettingsContext(this.cwd, this.projectRoot, this.isGitProject), parsed);
110
116
  return this.getHealth();
111
117
  }
112
118
  getActiveCount() {
@@ -347,21 +353,40 @@ export class RunManager {
347
353
  status: 400,
348
354
  };
349
355
  }
350
- return this.reserveAndStartPipeline(taskYaml, input.pipeline, `task file ${label}`, cwd, input.checkoutOverride);
356
+ return this.reserveAndStartPipeline(taskYaml, input.pipeline, `task file ${label}`, cwd, input.checkoutOverride, input.skipGates, {
357
+ gitSha: input.gitSha,
358
+ ciPrUrl: input.ciPrUrl,
359
+ ciJobUrl: input.ciJobUrl,
360
+ });
351
361
  }
352
362
  async rerun(runId) {
353
363
  const cwd = this.options.cwd ?? process.cwd();
354
- let pipelineId;
364
+ let pipeline;
355
365
  let taskYaml;
366
+ let rerunCwd = cwd;
367
+ let rerunProjectRoot;
356
368
  try {
357
369
  const meta = await this.options.store.readRunMeta(runId);
358
- pipelineId = meta.pipeline_id;
370
+ if (!meta.pipeline_path) {
371
+ return {
372
+ ok: false,
373
+ reason: `Run ${runId} is missing pipeline_path; re-run requires stored catalog locators.`,
374
+ status: 400,
375
+ };
376
+ }
377
+ pipeline = normalizeCatalogPath(meta.pipeline_path);
378
+ rerunCwd = meta.project_root
379
+ ? normalizeCatalogPath(meta.project_root)
380
+ : cwd;
381
+ rerunProjectRoot = meta.project_root
382
+ ? normalizeCatalogPath(meta.project_root)
383
+ : undefined;
359
384
  taskYaml = await this.options.store.readTaskYaml(runId);
360
385
  }
361
386
  catch {
362
387
  return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
363
388
  }
364
- return this.reserveAndStartPipeline(taskYaml, pipelineId, `run ${runId} task`, cwd);
389
+ return this.reserveAndStartPipeline(taskYaml, pipeline, `run ${runId} task`, rerunCwd, undefined, undefined, undefined, rerunProjectRoot);
365
390
  }
366
391
  async retryStage(runId, stageId) {
367
392
  const retryKey = waitKey(runId, stageId);
@@ -485,7 +510,6 @@ export class RunManager {
485
510
  }
486
511
  }
487
512
  async resumeViaSubprocess(runId, stageId, opaqueAnswer) {
488
- const cwd = this.options.cwd ?? process.cwd();
489
513
  const store = this.options.store;
490
514
  const launcher = this.stageProcessLauncher;
491
515
  if (!launcher) {
@@ -500,7 +524,7 @@ export class RunManager {
500
524
  const launchResult = await launcher.launch({
501
525
  runId,
502
526
  stageId,
503
- rootDir: cwd,
527
+ rootDir: this.projectRoot,
504
528
  mode: "resume",
505
529
  resumeAnswer: opaqueAnswer,
506
530
  attempt,
@@ -527,10 +551,7 @@ export class RunManager {
527
551
  catch {
528
552
  // downstream resume may read envelope from store
529
553
  }
530
- const meta = await store.readRunMeta(runId);
531
- const taskYaml = await store.readTaskYaml(runId);
532
- const task = loadTaskFromYaml(taskYaml, `run ${runId} task`);
533
- const loaded = await loadPipeline(meta.pipeline_id, { cwd });
554
+ const { meta, task, loaded } = await loadRunContext(store, runId, this.cwd);
534
555
  const rest = await resumeRun({
535
556
  prepared: {
536
557
  task,
@@ -538,7 +559,8 @@ export class RunManager {
538
559
  run: { runId, workspaceDir: store.getWorkspaceDir(runId) },
539
560
  agent: this.options.agent,
540
561
  store,
541
- cwd,
562
+ cwd: this.cwd,
563
+ projectRoot: this.projectRoot,
542
564
  checkoutRoot: meta.checkout_root,
543
565
  hitl: this.hitl,
544
566
  operatorCatalog: this.options.operatorCatalog,
@@ -549,7 +571,10 @@ export class RunManager {
549
571
  executionMode: this.executionMode,
550
572
  stageProcessLauncher: launcher,
551
573
  });
552
- return rest.ok ? { ok: true } : { ok: false, reason: rest.reason };
574
+ if (rest.outcome === "failed") {
575
+ return { ok: false, reason: rest.reason };
576
+ }
577
+ return { ok: true };
553
578
  }
554
579
  catch (err) {
555
580
  const reason = err instanceof Error
@@ -583,6 +608,7 @@ export class RunManager {
583
608
  executionMode: this.executionMode,
584
609
  stageProcessLauncher: this.stageProcessLauncher,
585
610
  cwd: this.cwd,
611
+ factoryCwd: this.projectRoot,
586
612
  maxActiveStagesPerRun: this.maxActiveStagesPerRun,
587
613
  operatorCatalog: this.options.operatorCatalog,
588
614
  });
@@ -591,7 +617,7 @@ export class RunManager {
591
617
  this.attachedWaiting.delete(`${runId}\0${stageId}`);
592
618
  }
593
619
  }
594
- async reserveAndStartPipeline(taskYaml, pipelineId, taskLabel, cwd, checkoutOverride) {
620
+ async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot) {
595
621
  let checkoutKey;
596
622
  try {
597
623
  const task = loadTaskFromYaml(taskYaml, taskLabel);
@@ -616,14 +642,19 @@ export class RunManager {
616
642
  agent: this.options.agent,
617
643
  store: this.options.store,
618
644
  taskYaml,
619
- pipeline: pipelineId,
645
+ pipeline,
620
646
  cwd,
647
+ projectRoot: projectRoot ?? this.projectRoot,
621
648
  checkoutOverride,
649
+ gitSha: ciIdentity?.gitSha,
650
+ ciPrUrl: ciIdentity?.ciPrUrl,
651
+ ciJobUrl: ciIdentity?.ciJobUrl,
622
652
  hitl: this.hitl,
623
653
  maxActiveStagesPerRun: this.maxActiveStagesPerRun,
624
654
  executionMode: this.executionMode,
625
655
  stageProcessLauncher: this.stageProcessLauncher,
626
656
  operatorCatalog: this.options.operatorCatalog,
657
+ skipGates,
627
658
  });
628
659
  this.track(reserved.provisionalId, started.runId, started.done);
629
660
  return { ok: true, runId: started.runId, done: started.done };