stageflow 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +158 -14
  2. package/dist/agent/activity.d.ts +10 -1
  3. package/dist/agent/activity.js +37 -2
  4. package/dist/agent/activityObserver.d.ts +5 -1
  5. package/dist/agent/activityObserver.js +174 -11
  6. package/dist/agent/fakeAgent.js +4 -0
  7. package/dist/agent/piAdapter.d.ts +16 -0
  8. package/dist/agent/piAdapter.js +88 -21
  9. package/dist/agent/port.d.ts +2 -0
  10. package/dist/agent/providerAuth.js +12 -7
  11. package/dist/catalog/displayCatalogPath.d.ts +12 -0
  12. package/dist/catalog/displayCatalogPath.js +28 -0
  13. package/dist/cli/initCommand.d.ts +9 -0
  14. package/dist/cli/initCommand.js +71 -0
  15. package/dist/cli/initTemplates.d.ts +3 -0
  16. package/dist/cli/initTemplates.js +19 -0
  17. package/dist/cli/operatorCatalog.d.ts +10 -0
  18. package/dist/cli/operatorCatalog.js +16 -0
  19. package/dist/cli/runCommand.d.ts +3 -1
  20. package/dist/cli/runCommand.js +48 -7
  21. package/dist/cli/validateCommand.d.ts +1 -1
  22. package/dist/cli/validateCommand.js +20 -3
  23. package/dist/cli.d.ts +1 -0
  24. package/dist/cli.js +52 -19
  25. package/dist/config/browseCatalog.d.ts +45 -0
  26. package/dist/config/browseCatalog.js +170 -0
  27. package/dist/config/createPipeline.d.ts +12 -3
  28. package/dist/config/createPipeline.js +292 -100
  29. package/dist/config/createStage.d.ts +10 -4
  30. package/dist/config/createStage.js +44 -14
  31. package/dist/config/listConfig.d.ts +14 -19
  32. package/dist/config/listConfig.js +26 -191
  33. package/dist/config/listModelsFromManifest.d.ts +2 -0
  34. package/dist/config/listModelsFromManifest.js +4 -0
  35. package/dist/config/loadOutcome.d.ts +4 -0
  36. package/dist/config/loadPipeline.d.ts +1 -15
  37. package/dist/config/loadPipeline.js +84 -142
  38. package/dist/config/loadStage.d.ts +4 -0
  39. package/dist/config/loadStage.js +60 -32
  40. package/dist/config/loadStageflowManifest.d.ts +5 -0
  41. package/dist/config/loadStageflowManifest.js +157 -0
  42. package/dist/config/mergePipelineIncludes.d.ts +9 -0
  43. package/dist/config/mergePipelineIncludes.js +141 -0
  44. package/dist/config/normalizePipelineStageEntry.d.ts +16 -0
  45. package/dist/config/normalizePipelineStageEntry.js +186 -0
  46. package/dist/config/pipelineStageKeys.d.ts +4 -0
  47. package/dist/config/pipelineStageKeys.js +14 -0
  48. package/dist/config/resolveCatalogContext.d.ts +13 -0
  49. package/dist/config/resolveCatalogContext.js +13 -0
  50. package/dist/config/resolveForkEmitContext.d.ts +3 -0
  51. package/dist/config/resolveForkEmitContext.js +12 -0
  52. package/dist/config/resolvePipelineDag.d.ts +5 -1
  53. package/dist/config/resolvePipelineDag.js +54 -36
  54. package/dist/config/scanCatalogPaths.d.ts +4 -0
  55. package/dist/config/scanCatalogPaths.js +83 -0
  56. package/dist/config/validateCatalog.d.ts +22 -4
  57. package/dist/config/validateCatalog.js +169 -132
  58. package/dist/envelope/check.js +7 -0
  59. package/dist/envelope/forkChoice.d.ts +3 -0
  60. package/dist/envelope/forkChoice.js +47 -0
  61. package/dist/index.d.ts +4 -0
  62. package/dist/index.js +4 -0
  63. package/dist/mcp/projectRun.d.ts +3 -0
  64. package/dist/mcp/projectRun.js +7 -0
  65. package/dist/mcp/tools.js +28 -11
  66. package/dist/project/findProjectRoot.d.ts +2 -0
  67. package/dist/project/findProjectRoot.js +51 -0
  68. package/dist/project/globalHome.d.ts +2 -0
  69. package/dist/project/globalHome.js +19 -0
  70. package/dist/project/resolveProjectContext.d.ts +9 -0
  71. package/dist/project/resolveProjectContext.js +38 -0
  72. package/dist/project/resolveStageflowContext.d.ts +11 -0
  73. package/dist/project/resolveStageflowContext.js +66 -0
  74. package/dist/runstore/normalizeCatalogPath.d.ts +1 -0
  75. package/dist/runstore/normalizeCatalogPath.js +4 -0
  76. package/dist/runstore/port.d.ts +9 -0
  77. package/dist/runstore/runProjection.js +11 -1
  78. package/dist/runstore/sqlite/SqliteRunStore.js +30 -4
  79. package/dist/runstore/sqlite/schema.d.ts +1 -1
  80. package/dist/runstore/sqlite/schema.js +4 -1
  81. package/dist/runtime/credentialBinding.d.ts +8 -6
  82. package/dist/runtime/credentialBinding.js +18 -24
  83. package/dist/runtime/pipelineRunner.d.ts +3 -0
  84. package/dist/runtime/pipelineRunner.js +15 -1
  85. package/dist/runtime/pipelineScheduler.d.ts +2 -0
  86. package/dist/runtime/pipelineScheduler.js +51 -11
  87. package/dist/runtime/reloadRunCatalog.d.ts +5 -0
  88. package/dist/runtime/reloadRunCatalog.js +44 -0
  89. package/dist/runtime/resumeReconstruct.d.ts +1 -0
  90. package/dist/runtime/resumeReconstruct.js +23 -7
  91. package/dist/runtime/runManager.d.ts +4 -0
  92. package/dist/runtime/runManager.js +36 -16
  93. package/dist/runtime/settingsFile.d.ts +12 -0
  94. package/dist/runtime/settingsFile.js +87 -6
  95. package/dist/runtime/stageAttemptBootstrap.js +3 -0
  96. package/dist/runtime/stageProcessLauncher.js +3 -0
  97. package/dist/runtime/stageRecovery.js +3 -1
  98. package/dist/runtime/stageWorker.js +2 -6
  99. package/dist/server/http.js +33 -9
  100. package/dist/tools/emitStageEnvelope.d.ts +3 -1
  101. package/dist/tools/emitStageEnvelope.js +11 -1
  102. package/dist/types/envelope.d.ts +1 -0
  103. package/dist/types/forkChoice.d.ts +8 -0
  104. package/dist/types/forkChoice.js +1 -0
  105. package/dist/types/pipeline.d.ts +51 -2
  106. package/dist/types/stageflowManifest.d.ts +23 -0
  107. package/dist/types/stageflowManifest.js +1 -0
  108. package/dist/ui/assets/index-CFSzDZje.js +118 -0
  109. package/dist/ui/assets/{index-DCsDopak.css → index-DefBlEvN.css} +1 -1
  110. package/dist/ui/index.html +3 -2
  111. package/dist/ui/stageflow-icon.svg +12 -0
  112. package/package.json +7 -5
  113. package/dist/ui/assets/index-Cry0Tpfx.js +0 -118
@@ -0,0 +1,38 @@
1
+ import os from "node:os";
2
+ import path from "node:path";
3
+ import { findProjectRoot } from "./findProjectRoot.js";
4
+ import { ensureGlobalHome, globalStageflowHome } from "./globalHome.js";
5
+ export function resolveProjectContext(invocationCwd) {
6
+ const resolvedInvocation = path.resolve(invocationCwd);
7
+ const gitRoot = findProjectRoot(resolvedInvocation);
8
+ const globalHome = ensureGlobalHome();
9
+ if (gitRoot !== null) {
10
+ return {
11
+ invocationCwd: resolvedInvocation,
12
+ projectRoot: gitRoot,
13
+ globalHome,
14
+ isGitProject: true,
15
+ };
16
+ }
17
+ return {
18
+ invocationCwd: resolvedInvocation,
19
+ projectRoot: os.homedir(),
20
+ globalHome,
21
+ isGitProject: false,
22
+ };
23
+ }
24
+ export function legacyProjectContext(cwd) {
25
+ const invocationCwd = path.resolve(cwd);
26
+ return {
27
+ invocationCwd,
28
+ projectRoot: invocationCwd,
29
+ globalHome: globalStageflowHome(),
30
+ isGitProject: false,
31
+ };
32
+ }
33
+ export function asProjectContext(input) {
34
+ if (typeof input !== "string") {
35
+ return input;
36
+ }
37
+ return resolveProjectContext(input);
38
+ }
@@ -0,0 +1,11 @@
1
+ import type { LoadedManifest } from "../types/stageflowManifest.js";
2
+ import type { LoadIssue } from "../config/loadOutcome.js";
3
+ import type { ProjectContext } from "./resolveProjectContext.js";
4
+ export type CatalogManifestStatus = "ok" | "missing" | "invalid" | "not_git";
5
+ export type StageflowContext = ProjectContext & {
6
+ manifest: LoadedManifest | null;
7
+ manifestStatus: CatalogManifestStatus;
8
+ manifestIssues: LoadIssue[];
9
+ };
10
+ export declare function projectContextFromStageflow(ctx: StageflowContext): ProjectContext;
11
+ export declare function resolveStageflowContext(invocationCwd: string): Promise<StageflowContext>;
@@ -0,0 +1,66 @@
1
+ import { access } from "node:fs/promises";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+ import { loadStageflowManifestOutcome, manifestPathForProject, } from "../config/loadStageflowManifest.js";
5
+ import { findProjectRoot } from "./findProjectRoot.js";
6
+ import { ensureGlobalHome } from "./globalHome.js";
7
+ export function projectContextFromStageflow(ctx) {
8
+ return {
9
+ invocationCwd: ctx.invocationCwd,
10
+ projectRoot: ctx.projectRoot,
11
+ globalHome: ctx.globalHome,
12
+ isGitProject: ctx.isGitProject,
13
+ };
14
+ }
15
+ export async function resolveStageflowContext(invocationCwd) {
16
+ const resolvedInvocation = path.resolve(invocationCwd);
17
+ const gitRoot = findProjectRoot(resolvedInvocation);
18
+ const globalHome = ensureGlobalHome();
19
+ if (gitRoot === null) {
20
+ return {
21
+ invocationCwd: resolvedInvocation,
22
+ projectRoot: os.homedir(),
23
+ globalHome,
24
+ isGitProject: false,
25
+ manifest: null,
26
+ manifestStatus: "not_git",
27
+ manifestIssues: [],
28
+ };
29
+ }
30
+ const manifestPath = manifestPathForProject(gitRoot);
31
+ try {
32
+ await access(manifestPath);
33
+ }
34
+ catch {
35
+ return {
36
+ invocationCwd: resolvedInvocation,
37
+ projectRoot: gitRoot,
38
+ globalHome,
39
+ isGitProject: true,
40
+ manifest: null,
41
+ manifestStatus: "missing",
42
+ manifestIssues: [],
43
+ };
44
+ }
45
+ const outcome = await loadStageflowManifestOutcome(gitRoot);
46
+ if (!outcome.ok) {
47
+ return {
48
+ invocationCwd: resolvedInvocation,
49
+ projectRoot: gitRoot,
50
+ globalHome,
51
+ isGitProject: true,
52
+ manifest: null,
53
+ manifestStatus: "invalid",
54
+ manifestIssues: outcome.issues,
55
+ };
56
+ }
57
+ return {
58
+ invocationCwd: resolvedInvocation,
59
+ projectRoot: gitRoot,
60
+ globalHome,
61
+ isGitProject: true,
62
+ manifest: outcome.value,
63
+ manifestStatus: "ok",
64
+ manifestIssues: [],
65
+ };
66
+ }
@@ -0,0 +1 @@
1
+ export declare function normalizeCatalogPath(raw: string): string;
@@ -0,0 +1,4 @@
1
+ import path from "node:path";
2
+ export function normalizeCatalogPath(raw) {
3
+ return path.resolve(raw);
4
+ }
@@ -40,6 +40,9 @@ export type RunMeta = {
40
40
  ci_pr_url?: string;
41
41
  ci_job_url?: string;
42
42
  pipeline_dag?: RunPipelineDagSnapshot;
43
+ pipeline_path?: string;
44
+ task_path?: string;
45
+ project_root?: string;
43
46
  };
44
47
  export type CreatedRun = {
45
48
  runId: string;
@@ -84,6 +87,9 @@ export type RunSummary = {
84
87
  run_id: string;
85
88
  pipeline_id: string;
86
89
  task_id?: string;
90
+ pipeline_path?: string;
91
+ task_path?: string;
92
+ project_root?: string;
87
93
  status: RunStatus;
88
94
  created_at: string;
89
95
  updated_at?: string;
@@ -115,6 +121,9 @@ export type CreateRunInput = {
115
121
  ciPrUrl?: string;
116
122
  ciJobUrl?: string;
117
123
  pipelineDag?: RunPipelineDagSnapshot;
124
+ pipelinePath?: string;
125
+ taskPath?: string;
126
+ projectRoot?: string;
118
127
  };
119
128
  /**
120
129
  * Persistence port for pipeline run state.
@@ -65,14 +65,24 @@ function failedFieldsFromStages(stages) {
65
65
  };
66
66
  }
67
67
  function resolveListedStatus(stages, meta) {
68
+ if (stages.length === 0)
69
+ return meta.status ?? "created";
68
70
  const derived = deriveStatusFromStages(stages);
69
- return stages.length > 0 ? derived : (meta.status ?? derived);
71
+ if (meta.status === "succeeded" && derived === "running") {
72
+ const hasActiveStage = stages.some((s) => s.status === "running" || s.status === "waiting_for_input");
73
+ if (!hasActiveStage)
74
+ return "succeeded";
75
+ }
76
+ return derived;
70
77
  }
71
78
  export function projectRunSummary(meta, stages) {
72
79
  return {
73
80
  run_id: meta.run_id,
74
81
  pipeline_id: meta.pipeline_id,
75
82
  task_id: meta.task_id,
83
+ ...(meta.pipeline_path !== undefined ? { pipeline_path: meta.pipeline_path } : {}),
84
+ ...(meta.task_path !== undefined ? { task_path: meta.task_path } : {}),
85
+ ...(meta.project_root !== undefined ? { project_root: meta.project_root } : {}),
76
86
  status: resolveListedStatus(stages, meta),
77
87
  created_at: meta.created_at,
78
88
  updated_at: meta.updated_at,
@@ -2,6 +2,7 @@ import Database from "better-sqlite3";
2
2
  import { mkdir } from "node:fs/promises";
3
3
  import path from "node:path";
4
4
  import { projectRunDetail, projectRunSummary, orderStageSnapshots } from "../runProjection.js";
5
+ import { normalizeCatalogPath } from "../normalizeCatalogPath.js";
5
6
  import { buildStageSnapshotFromStore } from "../stageSnapshot.js";
6
7
  import { parsePipelineDagSnapshot } from "../pipelineDagSnapshot.js";
7
8
  import { newRunId, runWorkspaceDir } from "../paths.js";
@@ -42,6 +43,15 @@ function ensurePipelineDagColumn(db) {
42
43
  db.exec(`ALTER TABLE runs ADD COLUMN pipeline_dag_json TEXT`);
43
44
  }
44
45
  }
46
+ function ensureRunLocatorColumns(db) {
47
+ const cols = db.prepare(`PRAGMA table_info(runs)`).all();
48
+ const names = new Set(cols.map((c) => c.name));
49
+ for (const name of ["pipeline_path", "task_path", "project_root"]) {
50
+ if (!names.has(name)) {
51
+ db.exec(`ALTER TABLE runs ADD COLUMN ${name} TEXT`);
52
+ }
53
+ }
54
+ }
45
55
  function ensureStageExecutionsTable(db) {
46
56
  db.exec(`
47
57
  CREATE TABLE IF NOT EXISTS stage_executions (
@@ -98,6 +108,7 @@ export class SqliteRunStore {
98
108
  ensureCheckoutRootColumn(this.db);
99
109
  ensureCiIdentityColumns(this.db);
100
110
  ensurePipelineDagColumn(this.db);
111
+ ensureRunLocatorColumns(this.db);
101
112
  ensureStageExecutionsTable(this.db);
102
113
  ensureStageEventsAttemptColumn(this.db);
103
114
  this.migratePromise = importDiskRunsIfEmpty(this.db, storeRoot).then(() => undefined);
@@ -115,11 +126,20 @@ export class SqliteRunStore {
115
126
  const workspaceDir = this.getWorkspaceDir(runId);
116
127
  await mkdir(path.join(workspaceDir, "stages"), { recursive: true });
117
128
  const now = new Date().toISOString();
129
+ const pipelinePath = input.pipelinePath
130
+ ? normalizeCatalogPath(input.pipelinePath)
131
+ : null;
132
+ const taskPath = input.taskPath
133
+ ? normalizeCatalogPath(input.taskPath)
134
+ : null;
135
+ const projectRoot = input.projectRoot
136
+ ? normalizeCatalogPath(input.projectRoot)
137
+ : null;
118
138
  this.db
119
139
  .prepare(`INSERT INTO runs
120
- (run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url)
140
+ (run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root)
121
141
  VALUES
122
- (@run_id, @pipeline_id, @task_id, @task_yaml, @status, @created_at, @updated_at, @checkout_root, @pipeline_dag_json, @git_sha, @ci_pr_url, @ci_job_url)`)
142
+ (@run_id, @pipeline_id, @task_id, @task_yaml, @status, @created_at, @updated_at, @checkout_root, @pipeline_dag_json, @git_sha, @ci_pr_url, @ci_job_url, @pipeline_path, @task_path, @project_root)`)
123
143
  .run({
124
144
  run_id: runId,
125
145
  pipeline_id: input.pipelineId,
@@ -135,6 +155,9 @@ export class SqliteRunStore {
135
155
  git_sha: input.gitSha ?? null,
136
156
  ci_pr_url: input.ciPrUrl ?? null,
137
157
  ci_job_url: input.ciJobUrl ?? null,
158
+ pipeline_path: pipelinePath,
159
+ task_path: taskPath,
160
+ project_root: projectRoot,
138
161
  });
139
162
  return { runId, workspaceDir };
140
163
  }
@@ -391,7 +414,7 @@ export class SqliteRunStore {
391
414
  async listRuns() {
392
415
  await this.ready();
393
416
  const rows = this.db
394
- .prepare(`SELECT run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url
417
+ .prepare(`SELECT run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root
395
418
  FROM runs ORDER BY created_at DESC`)
396
419
  .all();
397
420
  const summaries = [];
@@ -426,11 +449,14 @@ export class SqliteRunStore {
426
449
  ...(row.git_sha != null ? { git_sha: row.git_sha } : {}),
427
450
  ...(row.ci_pr_url != null ? { ci_pr_url: row.ci_pr_url } : {}),
428
451
  ...(row.ci_job_url != null ? { ci_job_url: row.ci_job_url } : {}),
452
+ ...(row.pipeline_path != null ? { pipeline_path: row.pipeline_path } : {}),
453
+ ...(row.task_path != null ? { task_path: row.task_path } : {}),
454
+ ...(row.project_root != null ? { project_root: row.project_root } : {}),
429
455
  };
430
456
  }
431
457
  getRunRow(runId) {
432
458
  const row = this.db
433
- .prepare(`SELECT run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url
459
+ .prepare(`SELECT run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root
434
460
  FROM runs WHERE run_id = ?`)
435
461
  .get(runId);
436
462
  if (!row)
@@ -1 +1 @@
1
- export declare const SCHEMA_SQL = "\nCREATE TABLE IF NOT EXISTS runs (\n run_id TEXT PRIMARY KEY,\n pipeline_id TEXT NOT NULL,\n task_id TEXT,\n task_yaml TEXT NOT NULL,\n status TEXT NOT NULL,\n created_at TEXT NOT NULL,\n updated_at TEXT NOT NULL,\n checkout_root TEXT,\n pipeline_dag_json TEXT,\n git_sha TEXT,\n ci_pr_url TEXT,\n ci_job_url TEXT\n);\n\nCREATE TABLE IF NOT EXISTS stages (\n run_id TEXT NOT NULL,\n stage_id TEXT NOT NULL,\n status TEXT,\n summary TEXT,\n envelope_json TEXT,\n started_at TEXT,\n finished_at TEXT,\n PRIMARY KEY (run_id, stage_id),\n FOREIGN KEY (run_id) REFERENCES runs(run_id)\n);\n\nCREATE TABLE IF NOT EXISTS stage_events (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n run_id TEXT NOT NULL,\n stage_id TEXT NOT NULL,\n at TEXT NOT NULL,\n event TEXT NOT NULL,\n payload_json TEXT,\n FOREIGN KEY (run_id) REFERENCES runs(run_id)\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_status_created\n ON runs (status, created_at DESC);\n\nCREATE INDEX IF NOT EXISTS idx_runs_pipeline_created\n ON runs (pipeline_id, created_at DESC);\n\nCREATE INDEX IF NOT EXISTS idx_stage_events_run_stage_at\n ON stage_events (run_id, stage_id, at);\n\nCREATE TABLE IF NOT EXISTS stage_executions (\n run_id TEXT NOT NULL,\n stage_id TEXT NOT NULL,\n attempt INTEGER NOT NULL,\n status TEXT NOT NULL,\n started_at TEXT,\n finished_at TEXT,\n envelope_json TEXT,\n PRIMARY KEY (run_id, stage_id, attempt),\n FOREIGN KEY (run_id) REFERENCES runs(run_id)\n);\n\nCREATE INDEX IF NOT EXISTS idx_stage_executions_run_stage\n ON stage_executions (run_id, stage_id, attempt);\n";
1
+ export declare const SCHEMA_SQL = "\nCREATE TABLE IF NOT EXISTS runs (\n run_id TEXT PRIMARY KEY,\n pipeline_id TEXT NOT NULL,\n task_id TEXT,\n task_yaml TEXT NOT NULL,\n status TEXT NOT NULL,\n created_at TEXT NOT NULL,\n updated_at TEXT NOT NULL,\n checkout_root TEXT,\n pipeline_dag_json TEXT,\n git_sha TEXT,\n ci_pr_url TEXT,\n ci_job_url TEXT,\n pipeline_path TEXT,\n task_path TEXT,\n project_root TEXT\n);\n\nCREATE TABLE IF NOT EXISTS stages (\n run_id TEXT NOT NULL,\n stage_id TEXT NOT NULL,\n status TEXT,\n summary TEXT,\n envelope_json TEXT,\n started_at TEXT,\n finished_at TEXT,\n PRIMARY KEY (run_id, stage_id),\n FOREIGN KEY (run_id) REFERENCES runs(run_id)\n);\n\nCREATE TABLE IF NOT EXISTS stage_events (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n run_id TEXT NOT NULL,\n stage_id TEXT NOT NULL,\n at TEXT NOT NULL,\n event TEXT NOT NULL,\n payload_json TEXT,\n FOREIGN KEY (run_id) REFERENCES runs(run_id)\n);\n\nCREATE INDEX IF NOT EXISTS idx_runs_status_created\n ON runs (status, created_at DESC);\n\nCREATE INDEX IF NOT EXISTS idx_runs_pipeline_created\n ON runs (pipeline_id, created_at DESC);\n\nCREATE INDEX IF NOT EXISTS idx_stage_events_run_stage_at\n ON stage_events (run_id, stage_id, at);\n\nCREATE TABLE IF NOT EXISTS stage_executions (\n run_id TEXT NOT NULL,\n stage_id TEXT NOT NULL,\n attempt INTEGER NOT NULL,\n status TEXT NOT NULL,\n started_at TEXT,\n finished_at TEXT,\n envelope_json TEXT,\n PRIMARY KEY (run_id, stage_id, attempt),\n FOREIGN KEY (run_id) REFERENCES runs(run_id)\n);\n\nCREATE INDEX IF NOT EXISTS idx_stage_executions_run_stage\n ON stage_executions (run_id, stage_id, attempt);\n";
@@ -11,7 +11,10 @@ CREATE TABLE IF NOT EXISTS runs (
11
11
  pipeline_dag_json TEXT,
12
12
  git_sha TEXT,
13
13
  ci_pr_url TEXT,
14
- ci_job_url TEXT
14
+ ci_job_url TEXT,
15
+ pipeline_path TEXT,
16
+ task_path TEXT,
17
+ project_root TEXT
15
18
  );
16
19
 
17
20
  CREATE TABLE IF NOT EXISTS stages (
@@ -1,5 +1,6 @@
1
- import { parseCredentialSource, readCredentialSourceFromFile, writeCredentialSourceToFile, type CredentialSource } from "./settingsFile.js";
2
- export { parseCredentialSource, readCredentialSourceFromFile, writeCredentialSourceToFile, };
1
+ import { type ProjectContext } from "../project/resolveProjectContext.js";
2
+ import { parseCredentialSource, readCredentialSourceFromContext, writeCredentialSourceToContext, type CredentialSource } from "./settingsFile.js";
3
+ export { parseCredentialSource, readCredentialSourceFromContext, writeCredentialSourceToContext, };
3
4
  export type CredentialBinding = {
4
5
  source: CredentialSource;
5
6
  authPath: string;
@@ -8,9 +9,10 @@ export type CredentialBinding = {
8
9
  export type ResolveCredentialBindingOptions = {
9
10
  piHomeAuthPath?: string;
10
11
  };
11
- export declare function sfOwnedAgentDir(cwd: string): string;
12
- export declare function sfOwnedAuthPath(cwd: string): string;
12
+ export declare function sfOwnedAgentDir(): string;
13
+ export declare function sfOwnedAuthPath(): string;
13
14
  export declare function piHomeAuthPath(override?: string): string;
14
15
  export declare function isUsableAuthFile(authPath: string): boolean;
15
- export declare function ensureSfOwnedAuthStore(cwd: string): string;
16
- export declare function resolveCredentialBinding(cwd: string, options?: ResolveCredentialBindingOptions): CredentialBinding;
16
+ export declare function ensureSfOwnedAuthStore(): string;
17
+ export declare function resolveCredentialBinding(ctx: ProjectContext | string, options?: ResolveCredentialBindingOptions): CredentialBinding;
18
+ export { readCredentialSourceFromFile, writeCredentialSourceToFile, } from "./settingsFile.js";
@@ -1,14 +1,15 @@
1
- import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
1
+ import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
2
2
  import os from "node:os";
3
3
  import path from "node:path";
4
- import { storeRootFor } from "../runstore/paths.js";
5
- import { parseCredentialSource, readCredentialSourceFromFile, writeCredentialSourceToFile, } from "./settingsFile.js";
6
- export { parseCredentialSource, readCredentialSourceFromFile, writeCredentialSourceToFile, };
7
- export function sfOwnedAgentDir(cwd) {
8
- return path.join(storeRootFor(cwd), "agent");
4
+ import { ensureGlobalHome } from "../project/globalHome.js";
5
+ import { resolveProjectContext, } from "../project/resolveProjectContext.js";
6
+ import { parseCredentialSource, readCredentialSourceFromContext, writeCredentialSourceToContext, } from "./settingsFile.js";
7
+ export { parseCredentialSource, readCredentialSourceFromContext, writeCredentialSourceToContext, };
8
+ export function sfOwnedAgentDir() {
9
+ return path.join(ensureGlobalHome(), "agent");
9
10
  }
10
- export function sfOwnedAuthPath(cwd) {
11
- return path.join(sfOwnedAgentDir(cwd), "auth.json");
11
+ export function sfOwnedAuthPath() {
12
+ return path.join(sfOwnedAgentDir(), "auth.json");
12
13
  }
13
14
  export function piHomeAuthPath(override) {
14
15
  return (override ??
@@ -25,18 +26,9 @@ export function isUsableAuthFile(authPath) {
25
26
  return false;
26
27
  }
27
28
  }
28
- export function ensureSfOwnedAuthStore(cwd) {
29
- const storeRoot = storeRootFor(cwd);
30
- mkdirSync(storeRoot, { recursive: true });
31
- const agentDir = sfOwnedAgentDir(cwd);
32
- mkdirSync(agentDir, { recursive: true });
33
- try {
34
- chmodSync(agentDir, 0o700);
35
- }
36
- catch {
37
- // best-effort on non-POSIX
38
- }
39
- const authPath = sfOwnedAuthPath(cwd);
29
+ export function ensureSfOwnedAuthStore() {
30
+ ensureGlobalHome();
31
+ const authPath = sfOwnedAuthPath();
40
32
  if (!existsSync(authPath)) {
41
33
  writeFileSync(authPath, "{}\n", { encoding: "utf8", mode: 0o600 });
42
34
  }
@@ -48,14 +40,15 @@ export function ensureSfOwnedAuthStore(cwd) {
48
40
  }
49
41
  return authPath;
50
42
  }
51
- export function resolveCredentialBinding(cwd, options = {}) {
43
+ export function resolveCredentialBinding(ctx, options = {}) {
44
+ const projectCtx = typeof ctx === "string" ? resolveProjectContext(ctx) : ctx;
52
45
  const piHome = piHomeAuthPath(options.piHomeAuthPath);
53
- const persisted = readCredentialSourceFromFile(cwd);
46
+ const persisted = readCredentialSourceFromContext(projectCtx);
54
47
  if (persisted !== undefined) {
55
48
  if (persisted === "sf_owned") {
56
49
  return {
57
50
  source: "sf_owned",
58
- authPath: ensureSfOwnedAuthStore(cwd),
51
+ authPath: ensureSfOwnedAuthStore(),
59
52
  provisional: false,
60
53
  };
61
54
  }
@@ -74,7 +67,8 @@ export function resolveCredentialBinding(cwd, options = {}) {
74
67
  }
75
68
  return {
76
69
  source: "sf_owned",
77
- authPath: ensureSfOwnedAuthStore(cwd),
70
+ authPath: ensureSfOwnedAuthStore(),
78
71
  provisional: true,
79
72
  };
80
73
  }
74
+ export { readCredentialSourceFromFile, writeCredentialSourceToFile, } from "./settingsFile.js";
@@ -31,6 +31,7 @@ export type PreparedPipeline = {
31
31
  agent: AgentPort;
32
32
  store: RunStore;
33
33
  cwd: string;
34
+ projectRoot: string;
34
35
  checkoutRoot?: string;
35
36
  hitl?: StageHitlController;
36
37
  executionMode?: StageExecutionMode;
@@ -55,6 +56,7 @@ export declare function runPipeline(options: {
55
56
  taskYaml?: string;
56
57
  pipeline: string;
57
58
  cwd?: string;
59
+ projectRoot?: string;
58
60
  checkoutOverride?: string;
59
61
  hitl?: StageHitlController;
60
62
  maxActiveStagesPerRun?: number;
@@ -71,6 +73,7 @@ export declare function startPipeline(options: {
71
73
  taskYaml?: string;
72
74
  pipeline: string;
73
75
  cwd?: string;
76
+ projectRoot?: string;
74
77
  checkoutOverride?: string;
75
78
  gitSha?: string;
76
79
  ciPrUrl?: string;
@@ -1,10 +1,11 @@
1
1
  import { readFile } from "node:fs/promises";
2
2
  import path from "node:path";
3
- import { loadPipelineValidated } from "../config/loadPipeline.js";
3
+ import { loadPipelineValidated } from "../config/validateCatalog.js";
4
4
  import { loadTaskFromYaml } from "../config/loadTask.js";
5
5
  import { buildValidationResult } from "../config/validateCatalog.js";
6
6
  import { resolveAndValidateCheckout } from "./stageRoots.js";
7
7
  import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
8
+ import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
8
9
  import { runPipelineDag } from "./pipelineScheduler.js";
9
10
  import { readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
10
11
  import { StageProcessLauncher } from "./stageProcessLauncher.js";
@@ -41,6 +42,11 @@ async function preparePipeline(options) {
41
42
  const label = options.taskPath ?? "task.yaml";
42
43
  const task = loadTaskFromYaml(taskYaml, `task file ${label}`);
43
44
  const checkoutRoot = await resolveAndValidateCheckout(task, options.checkoutOverride, options.cwd);
45
+ const pipelinePath = normalizeCatalogPath(loaded.pipelinePath);
46
+ const taskPath = options.taskPath
47
+ ? normalizeCatalogPath(path.resolve(options.cwd, options.taskPath))
48
+ : undefined;
49
+ const projectRoot = normalizeCatalogPath(options.projectRoot ?? options.cwd);
44
50
  const run = await options.store.createRun({
45
51
  pipelineId: loaded.pipeline.id,
46
52
  taskYaml,
@@ -50,6 +56,9 @@ async function preparePipeline(options) {
50
56
  ciPrUrl: options.ciPrUrl,
51
57
  ciJobUrl: options.ciJobUrl,
52
58
  pipelineDag: buildPipelineDagSnapshotFromLoaded(loaded),
59
+ pipelinePath,
60
+ taskPath,
61
+ projectRoot,
53
62
  });
54
63
  const executionMode = readStageExecutionMode(process.env, options.executionMode);
55
64
  const stageProcessLauncher = resolveStageProcessLauncher(executionMode, options.stageProcessLauncher);
@@ -60,6 +69,7 @@ async function preparePipeline(options) {
60
69
  agent: options.agent,
61
70
  store: options.store,
62
71
  cwd: options.cwd,
72
+ projectRoot: options.projectRoot ?? options.cwd,
63
73
  checkoutRoot,
64
74
  hitl: options.hitl,
65
75
  executionMode,
@@ -86,6 +96,7 @@ export async function executeStages(prepared, options) {
86
96
  }
87
97
  export async function runPipeline(options) {
88
98
  const cwd = options.cwd ?? process.cwd();
99
+ const projectRoot = options.projectRoot ?? cwd;
89
100
  const prepared = await preparePipeline({
90
101
  agent: options.agent,
91
102
  store: options.store,
@@ -93,6 +104,7 @@ export async function runPipeline(options) {
93
104
  taskYaml: options.taskYaml,
94
105
  pipeline: options.pipeline,
95
106
  cwd,
107
+ projectRoot,
96
108
  checkoutOverride: options.checkoutOverride,
97
109
  hitl: options.hitl,
98
110
  executionMode: options.executionMode,
@@ -109,6 +121,7 @@ export async function runPipeline(options) {
109
121
  /** Create the run immediately, then execute stages in the returned promise. */
110
122
  export async function startPipeline(options) {
111
123
  const cwd = options.cwd ?? process.cwd();
124
+ const projectRoot = options.projectRoot ?? cwd;
112
125
  const prepared = await preparePipeline({
113
126
  agent: options.agent,
114
127
  store: options.store,
@@ -116,6 +129,7 @@ export async function startPipeline(options) {
116
129
  taskYaml: options.taskYaml,
117
130
  pipeline: options.pipeline,
118
131
  cwd,
132
+ projectRoot,
119
133
  checkoutOverride: options.checkoutOverride,
120
134
  gitSha: options.gitSha,
121
135
  ciPrUrl: options.ciPrUrl,
@@ -18,6 +18,7 @@ type SchedulerPreparedPipeline = {
18
18
  agent: AgentPort;
19
19
  store: RunStore;
20
20
  cwd: string;
21
+ projectRoot?: string;
21
22
  checkoutRoot?: string;
22
23
  hitl?: StageHitlController;
23
24
  operatorCatalog?: OperatorCatalog;
@@ -86,6 +87,7 @@ export type RetryRunOptions = {
86
87
  onRetryRootTerminal?: OnRetryRootTerminal;
87
88
  };
88
89
  export declare function retryRun(options: RetryRunOptions): Promise<PipelineRunResult>;
90
+ export declare function applyForkSkipsFromEnvelopes(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>): void;
89
91
  export declare function applyRetryRootDelta(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>, retryRoots: Map<string, number>, stageId: string, attempt: number, clearSchedulingHalt?: () => void): void;
90
92
  export declare function hasWorkOutsideBranch(dag: ResolvedPipelineDag, branchRootId: string, states: Map<string, StageScheduleState>, ctx?: HasWorkOutsideBranchContext): boolean;
91
93
  export declare function runPipelineDag(options: RunPipelineDagOptions): Promise<PipelineRunResult>;
@@ -1,3 +1,4 @@
1
+ import { normalizeForkChoice } from "../envelope/forkChoice.js";
1
2
  import { collectDownstreamClosure, collectDownstreamStageIds, } from "./dagTraversal.js";
2
3
  import { buildCompletedEnvelopesFromRun, buildStageConfigById, } from "./envelopeRouting.js";
3
4
  import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
@@ -104,7 +105,10 @@ export async function resumeRun(options) {
104
105
  return state !== undefined && terminalState(state);
105
106
  });
106
107
  if (allTerminal) {
107
- const allSucceeded = dag.nodes.every((node) => hydrated.states.get(node.id) === "succeeded");
108
+ const allSucceeded = dag.nodes.every((node) => {
109
+ const s = hydrated.states.get(node.id);
110
+ return s === "succeeded" || s === "skipped";
111
+ });
108
112
  await prepared.store.updateRunStatus(prepared.run.runId, allSucceeded ? "succeeded" : "failed");
109
113
  return {
110
114
  ok: allSucceeded,
@@ -133,6 +137,33 @@ export async function retryRun(options) {
133
137
  onRetryRootTerminal: options.onRetryRootTerminal,
134
138
  });
135
139
  }
140
+ function applyForkChoiceToSchedule(dag, forkStageId, chosen, states, options) {
141
+ const skipPending = (stageId) => {
142
+ if (states.get(stageId) === "pending") {
143
+ states.set(stageId, "skipped");
144
+ options?.onSkip?.(stageId);
145
+ }
146
+ };
147
+ for (const childId of dag.childrenOf[forkStageId] ?? []) {
148
+ if (!chosen.has(childId)) {
149
+ skipPending(childId);
150
+ for (const desc of collectDownstreamStageIds(dag, childId)) {
151
+ skipPending(desc);
152
+ }
153
+ }
154
+ }
155
+ }
156
+ export function applyForkSkipsFromEnvelopes(dag, states, completedEnvelopes) {
157
+ for (const node of dag.nodes) {
158
+ if (!node.fork)
159
+ continue;
160
+ const envelope = completedEnvelopes.get(node.id);
161
+ if (!envelope)
162
+ continue;
163
+ const chosen = normalizeForkChoice(envelope.fork_choice, "stored");
164
+ applyForkChoiceToSchedule(dag, node.id, chosen, states);
165
+ }
166
+ }
136
167
  export function applyRetryRootDelta(dag, states, completedEnvelopes, retryRoots, stageId, attempt, clearSchedulingHalt) {
137
168
  retryRoots.set(stageId, attempt);
138
169
  const downstream = collectDownstreamClosure(dag, [stageId]);
@@ -206,7 +237,8 @@ function isRunnable(dag, stageId, states, schedulingHalted) {
206
237
  export async function runPipelineDag(options) {
207
238
  const { prepared, maxActiveStagesPerRun } = options;
208
239
  const executionMode = options.executionMode ?? "process";
209
- const { agent, task, loaded, run, store, checkoutRoot, hitl, cwd, } = prepared;
240
+ const { agent, task, loaded, run, store, checkoutRoot, hitl, cwd, projectRoot, } = prepared;
241
+ const factoryCwd = projectRoot ?? cwd ?? process.cwd();
210
242
  const dag = loaded.dag;
211
243
  const stageById = buildStageConfigById(loaded);
212
244
  let { states, completedEnvelopes, schedulingHalted, firstFailureReason } = options.initialSchedule ??
@@ -260,6 +292,7 @@ export async function runPipelineDag(options) {
260
292
  }
261
293
  }
262
294
  }
295
+ applyForkSkipsFromEnvelopes(dag, states, completedEnvelopes);
263
296
  let activeCount = 0;
264
297
  const inFlight = new Set();
265
298
  const notifyRetryRootTerminal = (stageId, outcome) => {
@@ -323,6 +356,13 @@ export async function runPipelineDag(options) {
323
356
  states.set(stageId, "succeeded");
324
357
  completedEnvelopes.set(stageId, envelope);
325
358
  notifyRetryRootTerminal(stageId, "succeeded");
359
+ const node = dag.nodes.find((n) => n.id === stageId);
360
+ if (node?.fork) {
361
+ const chosen = normalizeForkChoice(envelope.fork_choice, "stored");
362
+ applyForkChoiceToSchedule(dag, stageId, chosen, states, {
363
+ onSkip: (id) => notifyRetryRootTerminal(id, "skipped"),
364
+ });
365
+ }
326
366
  };
327
367
  const launchStage = async (stageId) => {
328
368
  const stage = stageById.get(stageId);
@@ -341,7 +381,7 @@ export async function runPipelineDag(options) {
341
381
  const launchResult = await launcher.launch({
342
382
  runId: run.runId,
343
383
  stageId,
344
- rootDir: cwd,
384
+ rootDir: factoryCwd,
345
385
  attempt,
346
386
  ...(prepared.operatorCatalog !== undefined
347
387
  ? { operatorCatalog: prepared.operatorCatalog }
@@ -353,9 +393,8 @@ export async function runPipelineDag(options) {
353
393
  const envelope = await store.readEnvelope(run.runId, stageId);
354
394
  onStageSuccess(stageId, envelope);
355
395
  }
356
- catch {
357
- states.set(stageId, "succeeded");
358
- notifyRetryRootTerminal(stageId, "succeeded");
396
+ catch (err) {
397
+ await onStageFailure(stageId, err instanceof Error ? err.message : String(err));
359
398
  }
360
399
  return;
361
400
  }
@@ -377,7 +416,7 @@ export async function runPipelineDag(options) {
377
416
  workspaceDir: run.workspaceDir,
378
417
  hitl,
379
418
  attemptCtx,
380
- factoryCwd: cwd,
419
+ factoryCwd,
381
420
  operatorCatalog: prepared.operatorCatalog,
382
421
  completedEnvelopes,
383
422
  skipGates: prepared.skipGates,
@@ -485,7 +524,10 @@ export async function runPipelineDag(options) {
485
524
  reason: firstFailureReason,
486
525
  };
487
526
  }
488
- const allSucceeded = dag.nodes.every((node) => states.get(node.id) === "succeeded");
527
+ const allSucceeded = dag.nodes.every((node) => {
528
+ const s = states.get(node.id);
529
+ return s === "succeeded" || s === "skipped";
530
+ });
489
531
  if (!allSucceeded) {
490
532
  if (retryContext === undefined) {
491
533
  await store.updateRunStatus(run.runId, "failed");
@@ -498,9 +540,7 @@ export async function runPipelineDag(options) {
498
540
  reason: firstFailureReason ?? "pipeline incomplete",
499
541
  };
500
542
  }
501
- if (retryContext === undefined) {
502
- await store.updateRunStatus(run.runId, "succeeded");
503
- }
543
+ await store.updateRunStatus(run.runId, "succeeded");
504
544
  return {
505
545
  ok: true,
506
546
  outcome: "succeeded",
@@ -0,0 +1,5 @@
1
+ import type { RunMeta } from "../runstore/port.js";
2
+ import type { LoadedPipeline } from "../types/pipeline.js";
3
+ import type { TaskFile } from "../types/task.js";
4
+ export declare function reloadPipelineForRun(meta: RunMeta): Promise<LoadedPipeline>;
5
+ export declare function reloadTaskForRun(meta: RunMeta): Promise<TaskFile>;