stageflow 0.19.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +16 -3
  2. package/dist/a2a/configDiscovery.d.ts +9 -0
  3. package/dist/a2a/configDiscovery.js +18 -0
  4. package/dist/a2a/contracts.d.ts +93 -0
  5. package/dist/a2a/contracts.js +163 -0
  6. package/dist/a2a/limits.d.ts +14 -0
  7. package/dist/a2a/limits.js +29 -0
  8. package/dist/a2a/registry.d.ts +44 -0
  9. package/dist/a2a/registry.js +159 -0
  10. package/dist/a2a/server.d.ts +29 -0
  11. package/dist/a2a/server.js +341 -0
  12. package/dist/a2a/service.d.ts +97 -0
  13. package/dist/a2a/service.js +496 -0
  14. package/dist/a2a/store.d.ts +106 -0
  15. package/dist/a2a/store.js +245 -0
  16. package/dist/agent/activity.d.ts +5 -0
  17. package/dist/agent/activityObserver.d.ts +3 -0
  18. package/dist/agent/activityObserver.js +6 -1
  19. package/dist/agent/piAdapter.js +7 -3
  20. package/dist/agent/port.d.ts +2 -0
  21. package/dist/agent/providerInspect.d.ts +19 -0
  22. package/dist/agent/providerInspect.js +35 -0
  23. package/dist/agent/streamLogRedact.d.ts +2 -0
  24. package/dist/agent/streamLogRedact.js +13 -0
  25. package/dist/cli/a2aCommand.d.ts +8 -0
  26. package/dist/cli/a2aCommand.js +131 -0
  27. package/dist/cli/artifactCommand.js +2 -2
  28. package/dist/cli/envelopeCommand.js +2 -2
  29. package/dist/cli/exportRunCommand.js +2 -2
  30. package/dist/cli/hostClient.d.ts +43 -0
  31. package/dist/cli/hostClient.js +182 -0
  32. package/dist/cli/providersCommand.js +5 -3
  33. package/dist/cli/runCommand.d.ts +6 -5
  34. package/dist/cli/runCommand.js +23 -31
  35. package/dist/cli/runOutput.d.ts +1 -1
  36. package/dist/cli/runStageCommand.d.ts +33 -0
  37. package/dist/cli/runStageCommand.js +407 -0
  38. package/dist/cli/runsCommand.d.ts +3 -7
  39. package/dist/cli/runsCommand.js +33 -94
  40. package/dist/cli.js +20 -5
  41. package/dist/config/describePipeline.d.ts +26 -0
  42. package/dist/config/describePipeline.js +37 -0
  43. package/dist/config/loadPipeline.d.ts +13 -1
  44. package/dist/config/loadPipeline.js +34 -4
  45. package/dist/config/mergePipelineIncludes.d.ts +11 -0
  46. package/dist/config/mergePipelineIncludes.js +75 -0
  47. package/dist/config/validateCatalog.d.ts +2 -2
  48. package/dist/config/validateCatalog.js +24 -17
  49. package/dist/mcp/catalogTools.js +285 -52
  50. package/dist/mcp/controlTools.js +77 -19
  51. package/dist/mcp/deps.d.ts +3 -0
  52. package/dist/mcp/projectMcpTools.d.ts +3 -0
  53. package/dist/mcp/projectMcpTools.js +25 -0
  54. package/dist/mcp/providerTools.d.ts +3 -0
  55. package/dist/mcp/providerTools.js +17 -0
  56. package/dist/mcp/readArtifact.d.ts +9 -0
  57. package/dist/mcp/readArtifact.js +11 -0
  58. package/dist/mcp/server.js +9 -1
  59. package/dist/mcp/tailStreamLog.d.ts +8 -0
  60. package/dist/mcp/tailStreamLog.js +28 -0
  61. package/dist/mcp/toolResults.d.ts +17 -0
  62. package/dist/mcp/toolResults.js +15 -0
  63. package/dist/mcp/tools.js +4 -0
  64. package/dist/mcp/waitRun.js +4 -4
  65. package/dist/mcp/waitingGates.d.ts +1 -0
  66. package/dist/mcp/waitingGates.js +3 -2
  67. package/dist/package-meta.d.ts +1 -1
  68. package/dist/package-meta.js +1 -1
  69. package/dist/projection/projectRun.d.ts +3 -0
  70. package/dist/projection/projectRun.js +7 -0
  71. package/dist/runstore/createStore.d.ts +10 -0
  72. package/dist/runstore/createStore.js +13 -4
  73. package/dist/runstore/port.d.ts +17 -2
  74. package/dist/runstore/sqlite/SqliteRunStore.d.ts +11 -1
  75. package/dist/runstore/sqlite/SqliteRunStore.js +81 -26
  76. package/dist/runstore/sqlite/schema.d.ts +1 -1
  77. package/dist/runstore/sqlite/schema.js +6 -0
  78. package/dist/runstore/submission.d.ts +11 -0
  79. package/dist/runstore/submission.js +7 -0
  80. package/dist/runstore/workspaceLayout.d.ts +1 -0
  81. package/dist/runstore/workspaceLayout.js +3 -0
  82. package/dist/runtime/feedbackLoopCoordinator.d.ts +9 -1
  83. package/dist/runtime/feedbackLoopCoordinator.js +50 -20
  84. package/dist/runtime/feedbackLoopDecision.js +33 -1
  85. package/dist/runtime/pipelineRunner.d.ts +5 -3
  86. package/dist/runtime/pipelineRunner.js +6 -1
  87. package/dist/runtime/pipelineScheduler.js +32 -15
  88. package/dist/runtime/reloadRunCatalog.js +7 -1
  89. package/dist/runtime/resolveEnvelopeRef.d.ts +30 -0
  90. package/dist/runtime/resolveEnvelopeRef.js +79 -0
  91. package/dist/runtime/resumeReconstruct.js +2 -2
  92. package/dist/runtime/runManager.d.ts +24 -2
  93. package/dist/runtime/runManager.js +83 -19
  94. package/dist/runtime/settingsFile.d.ts +2 -0
  95. package/dist/runtime/settingsFile.js +19 -0
  96. package/dist/runtime/stageAttemptBootstrap.d.ts +3 -0
  97. package/dist/runtime/stageAttemptBootstrap.js +8 -2
  98. package/dist/runtime/stageStreamLog.d.ts +17 -0
  99. package/dist/runtime/stageStreamLog.js +89 -0
  100. package/dist/runtime/stageWorker.js +2 -1
  101. package/dist/server/bootstrap.d.ts +2 -0
  102. package/dist/server/bootstrap.js +22 -4
  103. package/dist/server/createHttpHost.js +6 -0
  104. package/dist/server/ensureGlobalService.d.ts +35 -0
  105. package/dist/server/ensureGlobalService.js +138 -0
  106. package/dist/server/http.d.ts +20 -1
  107. package/dist/server/http.js +450 -402
  108. package/dist/server/mcpHost.js +7 -8
  109. package/dist/server/operatorResults.d.ts +10 -0
  110. package/dist/server/operatorResults.js +24 -0
  111. package/dist/server/providerRoutes.d.ts +0 -6
  112. package/dist/server/providerRoutes.js +10 -18
  113. package/dist/types/pipeline.d.ts +8 -0
  114. package/dist/ui/assets/{index-DMVdl1qB.js → index-Da5EHU6M.js} +5 -5
  115. package/dist/ui/index.html +1 -1
  116. package/package.json +2 -1
  117. package/skills/stageflow/references/control-surface.md +2 -2
  118. package/skills/stageflow-run/SKILL.md +27 -9
  119. package/skills/stageflow-run/references/mcp-call.md +1 -1
  120. package/skills/stageflow-run/scripts/mcp-call.mjs +9 -0
@@ -1,6 +1,8 @@
1
1
  import { randomUUID } from "node:crypto";
2
+ import { RunSubmissionExistsError } from "../runstore/submission.js";
2
3
  import { readFile, realpath } from "node:fs/promises";
3
4
  import path from "node:path";
5
+ import { findProjectRoot } from "../project/findProjectRoot.js";
4
6
  import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
5
7
  import { loadRunContext } from "./resumeReconstruct.js";
6
8
  import { assertTimedOutStageEligible, reconstructTimedOutAndContinue, } from "./resumeTimedOut.js";
@@ -12,7 +14,7 @@ import { resolveFeedbackLoopDecision, } from "./feedbackLoopDecision.js";
12
14
  import { RunRetryCoordinator, readRetryRootWaitTimeoutMs, } from "./runRetryCoordinator.js";
13
15
  import { readMaxActiveStageProcesses, readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
14
16
  import { StageProcessLauncher } from "./stageProcessLauncher.js";
15
- import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, projectSettingsContext, readMaxConcurrentFromContext, writeMaxConcurrentToContext, } from "./settingsFile.js";
17
+ import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, readMaxConcurrentFromGlobal, writeMaxConcurrentToGlobal, } from "./settingsFile.js";
16
18
  import { StageHitlController, waitKey, } from "./stageHitl.js";
17
19
  import { resolveAndValidateCheckout, } from "./stageRoots.js";
18
20
  import { orchestrateAnswerResume } from "./answerResume.js";
@@ -21,6 +23,12 @@ import { resumeSessionFilePath } from "./stageAttemptContext.js";
21
23
  import { resolveStartTaskInput } from "./taskInput.js";
22
24
  import { failStageAsInterrupted, syncRunStatusFromStages, } from "./stageRecovery.js";
23
25
  import { blocksGenericRetry, manualRecoveryEligibility, readManualRecoveryState, } from "./manualRecoveryState.js";
26
+ function existingSubmissionResult(existing, request) {
27
+ if (existing.requestHash !== request.requestHash) {
28
+ return { ok: false, status: 409, reason: "Submission key was already used for different input" };
29
+ }
30
+ return { ok: true, runId: existing.runId, reused: true };
31
+ }
24
32
  const DEFAULT_MAX_CONCURRENT = 3;
25
33
  const STARTUP_RECONCILE_REASON = "process_interrupted: no active worker (server restart)";
26
34
  const OPERATOR_ABANDON_REASON = "process_interrupted: operator abandoned stage";
@@ -44,6 +52,7 @@ async function toCheckoutLeaseKey(absPath) {
44
52
  }
45
53
  export class RunManager {
46
54
  options;
55
+ submissionsInFlight = new Map();
47
56
  active = new Map();
48
57
  checkoutLeases = new Map();
49
58
  provisionalIds = new Set();
@@ -94,10 +103,9 @@ export class RunManager {
94
103
  this.cwd = options.cwd ?? process.cwd();
95
104
  this.projectRoot = options.projectRoot ?? this.cwd;
96
105
  this.isGitProject = options.isGitProject ?? false;
97
- const settingsCtx = projectSettingsContext(this.cwd, this.projectRoot, this.isGitProject);
98
106
  this.maxConcurrent =
99
107
  options.maxConcurrent ??
100
- readMaxConcurrentFromContext(settingsCtx) ??
108
+ readMaxConcurrentFromGlobal() ??
101
109
  parseMaxConcurrent(process.env.STAGEFLOW_MAX_CONCURRENT_RUNS);
102
110
  this.maxActiveStagesPerRun = readMaxActiveStagesPerRun(process.env, options.maxActiveStagesPerRun);
103
111
  this.executionMode = readStageExecutionMode(process.env, options.executionMode);
@@ -119,7 +127,7 @@ export class RunManager {
119
127
  throw new Error(INVALID_SLOT_COUNT_MESSAGE);
120
128
  }
121
129
  this.maxConcurrent = parsed;
122
- writeMaxConcurrentToContext(projectSettingsContext(this.cwd, this.projectRoot, this.isGitProject), parsed);
130
+ writeMaxConcurrentToGlobal(parsed);
123
131
  return this.getHealth();
124
132
  }
125
133
  getActiveCount() {
@@ -348,8 +356,48 @@ export class RunManager {
348
356
  }
349
357
  return { ok: true, runId, stageId };
350
358
  }
351
- async startRun(input) {
359
+ async startRunOnce(input, submission) {
360
+ if (!submission.key.trim() || submission.key.length > 256 || !/^[a-f0-9]{64}$/.test(submission.requestHash)) {
361
+ return { ok: false, status: 400, reason: "Submission requires a key and SHA-256 request hash" };
362
+ }
363
+ const pending = this.submissionsInFlight.get(submission.key);
364
+ if (pending) {
365
+ if (pending.requestHash !== submission.requestHash) {
366
+ return { ok: false, status: 409, reason: "Submission key was already used for different input" };
367
+ }
368
+ const result = await pending.result;
369
+ return result.ok ? { ...result, reused: true } : result;
370
+ }
371
+ const result = this.startSubmittedRun(input, submission);
372
+ this.submissionsInFlight.set(submission.key, { requestHash: submission.requestHash, result });
373
+ try {
374
+ return await result;
375
+ }
376
+ finally {
377
+ this.submissionsInFlight.delete(submission.key);
378
+ }
379
+ }
380
+ async startSubmittedRun(input, submission) {
381
+ const existing = await this.options.store.getRunBySubmission(submission.key);
382
+ if (existing)
383
+ return existingSubmissionResult(existing, submission);
384
+ try {
385
+ const result = await this.startRun(input, submission);
386
+ return result.ok ? { ok: true, runId: result.runId, reused: false, done: result.done } : result;
387
+ }
388
+ catch (error) {
389
+ if (error instanceof RunSubmissionExistsError)
390
+ return existingSubmissionResult(error.submission, submission);
391
+ throw error;
392
+ }
393
+ }
394
+ async startRun(input, submission) {
352
395
  const cwd = this.options.cwd ?? process.cwd();
396
+ // An inline pipeline has no filesystem anchor to derive a project root from.
397
+ const derivedProjectRoot = typeof input.pipeline === "string"
398
+ ? (findProjectRoot(path.dirname(path.resolve(cwd, input.pipeline))) ??
399
+ this.projectRoot)
400
+ : this.projectRoot;
353
401
  let resolved;
354
402
  try {
355
403
  resolved = resolveStartTaskInput(input, cwd);
@@ -384,7 +432,7 @@ export class RunManager {
384
432
  gitSha: input.gitSha,
385
433
  ciPrUrl: input.ciPrUrl,
386
434
  ciJobUrl: input.ciJobUrl,
387
- }, undefined, resolved.kind === "path" ? resolved.taskPath : undefined);
435
+ }, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined, submission);
388
436
  }
389
437
  async rerun(runId) {
390
438
  const cwd = this.options.cwd ?? process.cwd();
@@ -456,6 +504,7 @@ export class RunManager {
456
504
  }
457
505
  const insertedForResume = !wasActive;
458
506
  try {
507
+ const runProjectRoot = detail.project_root ?? this.projectRoot;
459
508
  const done = reconstructTimedOutAndContinue({
460
509
  runId,
461
510
  stageId,
@@ -463,9 +512,9 @@ export class RunManager {
463
512
  store: this.options.store,
464
513
  hitl: this.hitl,
465
514
  executionMode: this.executionMode,
466
- cwd: this.cwd,
515
+ cwd: runProjectRoot,
467
516
  maxActiveStagesPerRun: this.maxActiveStagesPerRun,
468
- factoryCwd: this.projectRoot,
517
+ factoryCwd: runProjectRoot,
469
518
  ...(this.stageProcessLauncher !== undefined
470
519
  ? { stageProcessLauncher: this.stageProcessLauncher }
471
520
  : {}),
@@ -585,6 +634,7 @@ export class RunManager {
585
634
  };
586
635
  }
587
636
  try {
637
+ let meta;
588
638
  let orchestrationConflict = this.active.has(runId) &&
589
639
  !this.attachedWaiting.has(waitKey(runId, stageId));
590
640
  if (!this.retryCoordinator.isActive(runId)) {
@@ -607,7 +657,7 @@ export class RunManager {
607
657
  let detail;
608
658
  try {
609
659
  detail = await this.options.store.readRun(runId);
610
- await this.options.store.readRunMeta(runId);
660
+ meta = await this.options.store.readRunMeta(runId);
611
661
  }
612
662
  catch {
613
663
  this.retryInFlight.delete(retryKey);
@@ -626,12 +676,21 @@ export class RunManager {
626
676
  }
627
677
  }
628
678
  }
679
+ if (meta === undefined) {
680
+ try {
681
+ meta = await this.options.store.readRunMeta(runId);
682
+ }
683
+ catch {
684
+ this.retryInFlight.delete(retryKey);
685
+ return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
686
+ }
687
+ }
629
688
  const result = await this.retryCoordinator.retryStage({
630
689
  runId,
631
690
  stageId,
632
691
  store: this.options.store,
633
692
  agent: this.options.agent,
634
- cwd: this.cwd,
693
+ cwd: meta.project_root ?? this.projectRoot,
635
694
  operatorCatalog: this.options.operatorCatalog,
636
695
  maxActiveStagesPerRun: this.maxActiveStagesPerRun,
637
696
  executionMode: this.executionMode,
@@ -980,6 +1039,7 @@ export class RunManager {
980
1039
  };
981
1040
  }
982
1041
  const { meta, task, loaded } = await loadRunContext(this.options.store, runId, this.cwd);
1042
+ const runProjectRoot = meta.project_root ?? this.projectRoot;
983
1043
  const done = runPipelineDag({
984
1044
  prepared: {
985
1045
  task,
@@ -990,8 +1050,8 @@ export class RunManager {
990
1050
  },
991
1051
  agent: this.options.agent,
992
1052
  store: this.options.store,
993
- cwd: this.cwd,
994
- projectRoot: this.projectRoot,
1053
+ cwd: runProjectRoot,
1054
+ projectRoot: runProjectRoot,
995
1055
  checkoutRoot: meta.checkout_root,
996
1056
  hitl: this.hitl,
997
1057
  operatorCatalog: this.options.operatorCatalog,
@@ -1041,10 +1101,12 @@ export class RunManager {
1041
1101
  const sessionFilePath = resumeSessionFilePath(workspaceDir, stageId, attempt);
1042
1102
  const eventOptions = { attempt };
1043
1103
  try {
1104
+ const runMeta = await store.readRunMeta(runId);
1105
+ const runProjectRoot = runMeta.project_root ?? this.projectRoot;
1044
1106
  const launchResult = await launcher.launch({
1045
1107
  runId,
1046
1108
  stageId,
1047
- rootDir: this.projectRoot,
1109
+ rootDir: runProjectRoot,
1048
1110
  mode: "resume",
1049
1111
  resumeAnswer: opaqueAnswer,
1050
1112
  attempt,
@@ -1079,8 +1141,8 @@ export class RunManager {
1079
1141
  run: { runId, workspaceDir: store.getWorkspaceDir(runId) },
1080
1142
  agent: this.options.agent,
1081
1143
  store,
1082
- cwd: this.cwd,
1083
- projectRoot: this.projectRoot,
1144
+ cwd: meta.project_root ?? this.projectRoot,
1145
+ projectRoot: meta.project_root ?? this.projectRoot,
1084
1146
  checkoutRoot: meta.checkout_root,
1085
1147
  hitl: this.hitl,
1086
1148
  operatorCatalog: this.options.operatorCatalog,
@@ -1137,7 +1199,7 @@ export class RunManager {
1137
1199
  this.attachedWaiting.delete(`${runId}\0${stageId}`);
1138
1200
  }
1139
1201
  }
1140
- async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath) {
1202
+ async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath, submission) {
1141
1203
  let checkoutKey;
1142
1204
  try {
1143
1205
  const task = loadTaskFromYaml(taskYaml, taskLabel);
@@ -1159,6 +1221,7 @@ export class RunManager {
1159
1221
  return reserved.failure;
1160
1222
  try {
1161
1223
  const started = await startPipeline({
1224
+ submission,
1162
1225
  agent: this.options.agent,
1163
1226
  store: this.options.store,
1164
1227
  taskYaml,
@@ -1182,7 +1245,7 @@ export class RunManager {
1182
1245
  }
1183
1246
  catch (err) {
1184
1247
  this.clearReservation(reserved.provisionalId);
1185
- if (err instanceof PipelineValidationError) {
1248
+ if (err instanceof PipelineValidationError || err instanceof RunSubmissionExistsError) {
1186
1249
  throw err;
1187
1250
  }
1188
1251
  return {
@@ -1277,6 +1340,7 @@ export class RunManager {
1277
1340
  return false;
1278
1341
  }
1279
1342
  try {
1343
+ const runProjectRoot = meta.project_root ?? this.projectRoot;
1280
1344
  const { meta: loadedMeta, task, loaded, workspaceDir } = await loadRunContext(this.options.store, runId, this.cwd);
1281
1345
  const promise = runPipelineDag({
1282
1346
  prepared: {
@@ -1285,8 +1349,8 @@ export class RunManager {
1285
1349
  run: { runId, workspaceDir },
1286
1350
  agent: this.options.agent,
1287
1351
  store: this.options.store,
1288
- cwd: this.cwd,
1289
- projectRoot: this.projectRoot,
1352
+ cwd: runProjectRoot,
1353
+ projectRoot: runProjectRoot,
1290
1354
  checkoutRoot: loadedMeta.checkout_root,
1291
1355
  hitl: this.hitl,
1292
1356
  operatorCatalog: this.options.operatorCatalog,
@@ -23,6 +23,8 @@ export declare function readMaxConcurrentFromContext(ctx: ProjectContext): numbe
23
23
  export declare function readMaxConcurrentFromFile(cwd: string): number | undefined;
24
24
  export declare function writeMaxConcurrentToContext(ctx: ProjectContext, maxConcurrent: number): void;
25
25
  export declare function writeMaxConcurrentToFile(cwd: string, maxConcurrent: number): void;
26
+ export declare function readMaxConcurrentFromGlobal(): number | undefined;
27
+ export declare function writeMaxConcurrentToGlobal(maxConcurrent: number): void;
26
28
  export declare function readCredentialSourceFromFile(cwd: string): CredentialSource | undefined;
27
29
  export declare function writeCredentialSourceToContext(ctx: ProjectContext, credentialSource: CredentialSource): void;
28
30
  export declare function writeCredentialSourceToFile(cwd: string, credentialSource: CredentialSource): void;
@@ -150,6 +150,25 @@ export function writeMaxConcurrentToContext(ctx, maxConcurrent) {
150
150
  export function writeMaxConcurrentToFile(cwd, maxConcurrent) {
151
151
  writeFactorySettings(cwd, { maxConcurrent });
152
152
  }
153
+ export function readMaxConcurrentFromGlobal() {
154
+ return parseFactorySettingsFromRaw(readRawSettingsFromFile(globalSettingsFilePath())).maxConcurrent;
155
+ }
156
+ export function writeMaxConcurrentToGlobal(maxConcurrent) {
157
+ if (parseSlotCount(maxConcurrent) === undefined) {
158
+ throw new Error(INVALID_SLOT_COUNT_MESSAGE);
159
+ }
160
+ const current = parseFactorySettingsFromRaw(readRawSettingsFromFile(globalSettingsFilePath()));
161
+ const next = { ...current, maxConcurrent };
162
+ const out = {};
163
+ if (next.maxConcurrent !== undefined) {
164
+ out.maxConcurrent = next.maxConcurrent;
165
+ }
166
+ if (next.credentialSource !== undefined) {
167
+ out.credentialSource = next.credentialSource;
168
+ }
169
+ mkdirSync(path.dirname(globalSettingsFilePath()), { recursive: true });
170
+ writeFileSync(globalSettingsFilePath(), `${JSON.stringify(out, null, 2)}\n`, "utf8");
171
+ }
153
172
  export function readCredentialSourceFromFile(cwd) {
154
173
  return readFactorySettings(cwd).credentialSource;
155
174
  }
@@ -1,4 +1,5 @@
1
1
  import type { AgentPort, FeedbackLoopContext, StageHandle, StageRunInput, StageSessionMode } from "../agent/port.js";
2
+ import { type StageStreamLogWriter } from "./stageStreamLog.js";
2
3
  import type { RunStore } from "../runstore/port.js";
3
4
  import type { StageEnvelope } from "../types/envelope.js";
4
5
  import type { ResolvedPipelineDag } from "../types/pipeline.js";
@@ -23,6 +24,8 @@ export type StageAttemptOpenInput = {
23
24
  attemptCtx?: StageAttemptContext;
24
25
  operatorCatalog?: OperatorCatalog;
25
26
  onActivity?: StageRunInput["onActivity"];
27
+ /** Test seam; defaults to the real createStageStreamLogWriter. */
28
+ streamLogWriterFactory?: (streamLogPath: string) => StageStreamLogWriter;
26
29
  roots?: StageRoots;
27
30
  resumeToken?: string;
28
31
  sessionMode?: StageSessionMode;
@@ -1,7 +1,8 @@
1
1
  import { resolveSkillByName } from "../config/listSkills.js";
2
2
  import { mkdir } from "node:fs/promises";
3
3
  import { MCP_CATALOG_FILENAME, STAGEFLOW_STAGE_ARTIFACTS_DIR_ENV, StageMcpError, resolveStageMcpServers, stampStagePromptArtifactsDir, } from "../config/resolveStageMcpServers.js";
4
- import { attemptArtifactsDir } from "../runstore/workspaceLayout.js";
4
+ import { attemptArtifactsDir, attemptStreamLogPath } from "../runstore/workspaceLayout.js";
5
+ import { createStageStreamLogWriter, } from "./stageStreamLog.js";
5
6
  import { resolveCloneEmitContext, resolveForkEmitContext } from "../config/resolveForkEmitContext.js";
6
7
  import { createAttemptQaTrailReader } from "../hitl/qaTrail.js";
7
8
  import { assertPriorInputPayload, payloadInstanceMismatch, } from "../envelope/payloadSchema.js";
@@ -199,6 +200,7 @@ export async function openStageAttempt(input) {
199
200
  return priorInput;
200
201
  const roots = resolveAttemptRoots(input, stageId);
201
202
  const attempt = input.attemptCtx?.attempt ?? 1;
203
+ const streamWriter = (input.streamLogWriterFactory ?? createStageStreamLogWriter)(attemptStreamLogPath(input.workspaceDir, stageId, attempt));
202
204
  const resumeToken = input.resumeToken ??
203
205
  resumeSessionFilePath(input.workspaceDir, stageId, attempt);
204
206
  const forkEmitContext = resolveForkEmitContext(input.dag, definitionId);
@@ -231,7 +233,11 @@ export async function openStageAttempt(input) {
231
233
  : {}),
232
234
  resumeToken,
233
235
  ...(input.sessionMode !== undefined ? { sessionMode: input.sessionMode } : {}),
234
- onActivity: input.onActivity,
236
+ onActivity: (event) => {
237
+ input.onActivity?.(event);
238
+ void streamWriter.flush();
239
+ },
240
+ onAssistantTextDelta: streamWriter.onDelta,
235
241
  forkEmitContext,
236
242
  cloneEmitContext,
237
243
  ...(feedbackLoopEmitContext !== undefined ? { feedbackLoopEmitContext } : {}),
@@ -0,0 +1,17 @@
1
+ /** First line of every stream.log: the logical byte offset of the content that follows. */
2
+ export declare const STREAM_LOG_HEADER_PREFIX = "BASE:";
3
+ export declare function encodeStreamLogHeader(base: number): string;
4
+ /** Split a stream.log buffer into its logical base offset and content bytes. */
5
+ export declare function parseStreamLogHeader(raw: Buffer): {
6
+ base: number;
7
+ headerLength: number;
8
+ };
9
+ export type StageStreamLogWriter = {
10
+ onDelta(delta: string): void;
11
+ /** Idempotent; resolves once any pending buffered text has been written to disk. */
12
+ flush(): Promise<void>;
13
+ };
14
+ export declare function createStageStreamLogWriter(streamLogPath: string, options?: {
15
+ flushThrottleMs?: number;
16
+ capBytes?: number;
17
+ }): StageStreamLogWriter;
@@ -0,0 +1,89 @@
1
+ import { mkdir, readFile, writeFile } from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { STREAM_FLUSH_THROTTLE_MS } from "../agent/activityObserver.js";
4
+ import { redactSecrets } from "../agent/streamLogRedact.js";
5
+ const DEFAULT_CAP_BYTES = 256 * 1024;
6
+ /** First line of every stream.log: the logical byte offset of the content that follows. */
7
+ export const STREAM_LOG_HEADER_PREFIX = "BASE:";
8
+ export function encodeStreamLogHeader(base) {
9
+ return `${STREAM_LOG_HEADER_PREFIX}${base}\n`;
10
+ }
11
+ /** Split a stream.log buffer into its logical base offset and content bytes. */
12
+ export function parseStreamLogHeader(raw) {
13
+ const prefix = Buffer.from(STREAM_LOG_HEADER_PREFIX, "utf8");
14
+ if (raw.subarray(0, prefix.length).equals(prefix)) {
15
+ const newlineIdx = raw.indexOf(0x0a);
16
+ if (newlineIdx !== -1) {
17
+ const baseStr = raw.subarray(prefix.length, newlineIdx).toString("utf8");
18
+ const base = Number(baseStr);
19
+ if (Number.isFinite(base) && base >= 0) {
20
+ return { base, headerLength: newlineIdx + 1 };
21
+ }
22
+ }
23
+ }
24
+ return { base: 0, headerLength: 0 };
25
+ }
26
+ async function appendChunk(streamLogPath, chunk, capBytes) {
27
+ await mkdir(path.dirname(streamLogPath), { recursive: true });
28
+ let base = 0;
29
+ let contentBytes;
30
+ try {
31
+ const raw = await readFile(streamLogPath);
32
+ const parsed = parseStreamLogHeader(raw);
33
+ base = parsed.base;
34
+ contentBytes = raw.subarray(parsed.headerLength);
35
+ }
36
+ catch {
37
+ contentBytes = Buffer.alloc(0);
38
+ }
39
+ contentBytes = Buffer.concat([contentBytes, Buffer.from(chunk, "utf8")]);
40
+ const headerBytes = Buffer.from(encodeStreamLogHeader(base), "utf8");
41
+ if (headerBytes.length + contentBytes.length > capBytes) {
42
+ let dropBytes = Math.floor(contentBytes.length / 2);
43
+ // Never split a multi-byte UTF-8 codepoint: skip past continuation bytes.
44
+ while (dropBytes < contentBytes.length &&
45
+ (contentBytes[dropBytes] & 0xc0) === 0x80) {
46
+ dropBytes++;
47
+ }
48
+ base += dropBytes;
49
+ contentBytes = contentBytes.subarray(dropBytes);
50
+ }
51
+ const finalHeader = Buffer.from(encodeStreamLogHeader(base), "utf8");
52
+ await writeFile(streamLogPath, Buffer.concat([finalHeader, contentBytes]));
53
+ }
54
+ export function createStageStreamLogWriter(streamLogPath, options = {}) {
55
+ const flushThrottleMs = options.flushThrottleMs ?? STREAM_FLUSH_THROTTLE_MS;
56
+ const capBytes = options.capBytes ?? DEFAULT_CAP_BYTES;
57
+ let buffer = "";
58
+ let timer = null;
59
+ // Serializes disk writes so the throttle timer and an eager flush() never race.
60
+ let writeChain = Promise.resolve();
61
+ function flushNow() {
62
+ if (timer !== null) {
63
+ clearTimeout(timer);
64
+ timer = null;
65
+ }
66
+ if (!buffer)
67
+ return writeChain;
68
+ const chunk = redactSecrets(buffer);
69
+ buffer = "";
70
+ writeChain = writeChain.then(() => appendChunk(streamLogPath, chunk, capBytes));
71
+ return writeChain;
72
+ }
73
+ return {
74
+ onDelta(delta) {
75
+ if (!delta)
76
+ return;
77
+ buffer += delta;
78
+ if (timer === null) {
79
+ timer = setTimeout(() => {
80
+ timer = null;
81
+ void flushNow();
82
+ }, flushThrottleMs);
83
+ }
84
+ },
85
+ flush() {
86
+ return flushNow();
87
+ },
88
+ };
89
+ }
@@ -3,6 +3,7 @@ import { asAgentBackendId } from "../agent/agentBackend.js";
3
3
  import { loadStageflowManifestOutcome } from "../config/loadStageflowManifest.js";
4
4
  import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
5
5
  import { createRunStore } from "../runstore/createStore.js";
6
+ import { globalStageflowHome } from "../project/globalHome.js";
6
7
  import { loadRunContext } from "./resumeReconstruct.js";
7
8
  import { bindPiAgentDirEnv, rootsForStageWorker, withResolvedAuthPath, } from "./stageRoots.js";
8
9
  import { runStage, isRunStageWaiting, } from "./stageRunner.js";
@@ -11,7 +12,7 @@ import { attemptContext } from "./stageAttemptContext.js";
11
12
  import { loadActiveFeedbackLoopContext } from "./feedbackLoopCoordinator.js";
12
13
  import { outcomeToWorkerResult, STAGE_WORKER_EXIT, } from "./stageWorkerProtocol.js";
13
14
  export async function runStageWorker(input) {
14
- const store = createRunStore({ rootDir: input.rootDir });
15
+ const store = createRunStore({ rootDir: globalStageflowHome() });
15
16
  const { meta, task, loaded } = await loadRunContext(store, input.runId, input.rootDir);
16
17
  const definitionId = definitionIdForInstance(meta.pipeline_dag, input.stageId);
17
18
  const stage = loaded.stages.find((s) => s.id === definitionId);
@@ -1,3 +1,4 @@
1
+ import { type A2aHost } from "../a2a/server.js";
1
2
  import type { AgentPort } from "../agent/port.js";
2
3
  import type { ProviderAuthContext } from "../agent/providerAuth.js";
3
4
  import { type RunStoreKind } from "../runstore/createStore.js";
@@ -18,6 +19,7 @@ export type StageflowHostOptions = {
18
19
  runChangeBus?: RunChangeBus;
19
20
  };
20
21
  export type StageflowHostBootstrap = {
22
+ a2a?: A2aHost;
21
23
  cwd: string;
22
24
  agentDir: string;
23
25
  rootDir: string;
@@ -1,5 +1,7 @@
1
1
  import { getAgentDir } from "@earendil-works/pi-coding-agent";
2
- import { createRunStore } from "../runstore/createStore.js";
2
+ import { createA2aHost } from "../a2a/server.js";
3
+ import { createRunStoreWithConnection } from "../runstore/createStore.js";
4
+ import { resolveA2aConfigPath } from "../a2a/configDiscovery.js";
3
5
  import { resolveStageflowContext } from "../project/resolveStageflowContext.js";
4
6
  import { findProjectRoot } from "../project/findProjectRoot.js";
5
7
  import { RunManager } from "../runtime/runManager.js";
@@ -14,8 +16,16 @@ export async function bootstrapStageflowHost(options) {
14
16
  const isGitProject = options.rootDir !== undefined
15
17
  ? findProjectRoot(rootDir) !== null
16
18
  : ctx.isGitProject;
17
- const rawStore = options.store ??
18
- createRunStore({ rootDir, kind: options.storeKind });
19
+ let rawStore;
20
+ let sqliteConnection;
21
+ if (options.store) {
22
+ rawStore = options.store;
23
+ }
24
+ else {
25
+ const created = createRunStoreWithConnection({ rootDir: ctx.globalHome, kind: options.storeKind });
26
+ rawStore = created.store;
27
+ sqliteConnection = created.connection;
28
+ }
19
29
  const boundBus = isRunStoreWrapped(rawStore)
20
30
  ? getRunChangeBusFromWrappedStore(rawStore)
21
31
  : undefined;
@@ -41,8 +51,16 @@ export async function bootstrapStageflowHost(options) {
41
51
  const mcpStateless = resolveMcpStateless({
42
52
  mcpStateless: options.mcpStateless,
43
53
  });
44
- const mcpHandler = createMcpHttpHandler({ manager, store, cwd, runChangeBus }, { mcpStateless });
54
+ const mcpHandler = createMcpHttpHandler({
55
+ manager,
56
+ store,
57
+ cwd,
58
+ runChangeBus,
59
+ providerAuthContext: options.providerAuthContext,
60
+ projectRoot: rootDir,
61
+ }, { mcpStateless });
45
62
  return {
63
+ a2a: await createA2aHost({ manager, runStore: store, rootDir, connection: sqliteConnection }, resolveA2aConfigPath(rootDir)),
46
64
  cwd,
47
65
  agentDir,
48
66
  rootDir,
@@ -17,6 +17,12 @@ export async function createHttpHost(options) {
17
17
  const method = req.method ?? "GET";
18
18
  const url = new URL(req.url ?? "/", `http://${host}:${port}`);
19
19
  const pathname = url.pathname;
20
+ if (pathname === "/api/a2a/status" && method === "GET") {
21
+ json(res, 200, boot.a2a?.status ?? { state: "disabled" });
22
+ return;
23
+ }
24
+ if (await boot.a2a?.handle(req, res, pathname))
25
+ return;
20
26
  if (pathname === "/mcp") {
21
27
  if (!validateMcpHost(req, res) || !validateMcpOrigin(req, res)) {
22
28
  return;
@@ -0,0 +1,35 @@
1
+ import { type ChildProcess, spawn as spawnDefault } from "node:child_process";
2
+ /** Coarse up/down probe result, used by CLI code that only needs "is it safe to talk to the host." */
3
+ export type HostProbeResult = "up" | "down";
4
+ /**
5
+ * Finer-grained probe result used to auto-start: "unreachable" (nothing
6
+ * listening, safe to spawn) is distinct from "unhealthy" (something is
7
+ * listening on the port but didn't answer with valid health JSON, so it's
8
+ * almost certainly not our service and spawning would just collide).
9
+ */
10
+ export type ServiceProbeResult = "up" | "unreachable" | "unhealthy";
11
+ export declare function hostBaseUrl(): string;
12
+ export declare function probeGlobalServiceDetailed(): Promise<ServiceProbeResult>;
13
+ export declare function defaultProbeHost(): Promise<HostProbeResult>;
14
+ export type EnsureGlobalServiceResult = {
15
+ ok: true;
16
+ alreadyRunning: boolean;
17
+ } | {
18
+ ok: false;
19
+ reason: "port_occupied" | "spawn_failed" | "timed_out";
20
+ message: string;
21
+ };
22
+ export interface EnsureGlobalServiceOptions {
23
+ /** Path to the CLI entry script to spawn `<entry> mcp` against. Defaults to the running process's own entry (process.argv[1]). */
24
+ cliEntry?: string;
25
+ probeHost?: () => Promise<ServiceProbeResult>;
26
+ spawnFn?: (command: string, args: string[], options: Parameters<typeof spawnDefault>[2]) => ChildProcess;
27
+ pollIntervalMs?: number;
28
+ timeoutMs?: number;
29
+ }
30
+ /**
31
+ * Health-probe-then-spawn: makes sure a global Stageflow service is
32
+ * reachable at the well-known port, starting one headlessly (`<cliEntry>
33
+ * mcp`, detached) if nothing answers yet. No-ops if one is already up.
34
+ */
35
+ export declare function ensureGlobalService(options?: EnsureGlobalServiceOptions): Promise<EnsureGlobalServiceResult>;