stageflow 0.19.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -3
- package/dist/a2a/configDiscovery.d.ts +9 -0
- package/dist/a2a/configDiscovery.js +18 -0
- package/dist/a2a/contracts.d.ts +93 -0
- package/dist/a2a/contracts.js +163 -0
- package/dist/a2a/limits.d.ts +14 -0
- package/dist/a2a/limits.js +29 -0
- package/dist/a2a/registry.d.ts +44 -0
- package/dist/a2a/registry.js +159 -0
- package/dist/a2a/server.d.ts +29 -0
- package/dist/a2a/server.js +341 -0
- package/dist/a2a/service.d.ts +97 -0
- package/dist/a2a/service.js +496 -0
- package/dist/a2a/store.d.ts +106 -0
- package/dist/a2a/store.js +245 -0
- package/dist/agent/activity.d.ts +5 -0
- package/dist/agent/activityObserver.d.ts +3 -0
- package/dist/agent/activityObserver.js +6 -1
- package/dist/agent/piAdapter.js +7 -3
- package/dist/agent/port.d.ts +2 -0
- package/dist/agent/providerInspect.d.ts +19 -0
- package/dist/agent/providerInspect.js +35 -0
- package/dist/agent/streamLogRedact.d.ts +2 -0
- package/dist/agent/streamLogRedact.js +13 -0
- package/dist/cli/a2aCommand.d.ts +8 -0
- package/dist/cli/a2aCommand.js +131 -0
- package/dist/cli/artifactCommand.js +2 -2
- package/dist/cli/envelopeCommand.js +2 -2
- package/dist/cli/exportRunCommand.js +2 -2
- package/dist/cli/hostClient.d.ts +43 -0
- package/dist/cli/hostClient.js +182 -0
- package/dist/cli/providersCommand.js +5 -3
- package/dist/cli/runCommand.d.ts +6 -5
- package/dist/cli/runCommand.js +23 -31
- package/dist/cli/runOutput.d.ts +1 -1
- package/dist/cli/runStageCommand.d.ts +33 -0
- package/dist/cli/runStageCommand.js +407 -0
- package/dist/cli/runsCommand.d.ts +3 -7
- package/dist/cli/runsCommand.js +33 -94
- package/dist/cli.js +20 -5
- package/dist/config/describePipeline.d.ts +26 -0
- package/dist/config/describePipeline.js +37 -0
- package/dist/config/loadPipeline.d.ts +13 -1
- package/dist/config/loadPipeline.js +34 -4
- package/dist/config/mergePipelineIncludes.d.ts +11 -0
- package/dist/config/mergePipelineIncludes.js +75 -0
- package/dist/config/validateCatalog.d.ts +2 -2
- package/dist/config/validateCatalog.js +24 -17
- package/dist/mcp/catalogTools.js +285 -52
- package/dist/mcp/controlTools.js +77 -19
- package/dist/mcp/deps.d.ts +3 -0
- package/dist/mcp/projectMcpTools.d.ts +3 -0
- package/dist/mcp/projectMcpTools.js +25 -0
- package/dist/mcp/providerTools.d.ts +3 -0
- package/dist/mcp/providerTools.js +17 -0
- package/dist/mcp/readArtifact.d.ts +9 -0
- package/dist/mcp/readArtifact.js +11 -0
- package/dist/mcp/server.js +9 -1
- package/dist/mcp/tailStreamLog.d.ts +8 -0
- package/dist/mcp/tailStreamLog.js +28 -0
- package/dist/mcp/toolResults.d.ts +17 -0
- package/dist/mcp/toolResults.js +15 -0
- package/dist/mcp/tools.js +4 -0
- package/dist/mcp/waitRun.js +4 -4
- package/dist/mcp/waitingGates.d.ts +1 -0
- package/dist/mcp/waitingGates.js +3 -2
- package/dist/package-meta.d.ts +1 -1
- package/dist/package-meta.js +1 -1
- package/dist/projection/projectRun.d.ts +3 -0
- package/dist/projection/projectRun.js +7 -0
- package/dist/runstore/createStore.d.ts +10 -0
- package/dist/runstore/createStore.js +13 -4
- package/dist/runstore/port.d.ts +17 -2
- package/dist/runstore/sqlite/SqliteRunStore.d.ts +11 -1
- package/dist/runstore/sqlite/SqliteRunStore.js +81 -26
- package/dist/runstore/sqlite/schema.d.ts +1 -1
- package/dist/runstore/sqlite/schema.js +6 -0
- package/dist/runstore/submission.d.ts +11 -0
- package/dist/runstore/submission.js +7 -0
- package/dist/runstore/workspaceLayout.d.ts +1 -0
- package/dist/runstore/workspaceLayout.js +3 -0
- package/dist/runtime/feedbackLoopCoordinator.d.ts +9 -1
- package/dist/runtime/feedbackLoopCoordinator.js +50 -20
- package/dist/runtime/feedbackLoopDecision.js +33 -1
- package/dist/runtime/pipelineRunner.d.ts +5 -3
- package/dist/runtime/pipelineRunner.js +6 -1
- package/dist/runtime/pipelineScheduler.js +32 -15
- package/dist/runtime/reloadRunCatalog.js +7 -1
- package/dist/runtime/resolveEnvelopeRef.d.ts +30 -0
- package/dist/runtime/resolveEnvelopeRef.js +79 -0
- package/dist/runtime/resumeReconstruct.js +2 -2
- package/dist/runtime/runManager.d.ts +24 -2
- package/dist/runtime/runManager.js +83 -19
- package/dist/runtime/settingsFile.d.ts +2 -0
- package/dist/runtime/settingsFile.js +19 -0
- package/dist/runtime/stageAttemptBootstrap.d.ts +3 -0
- package/dist/runtime/stageAttemptBootstrap.js +8 -2
- package/dist/runtime/stageStreamLog.d.ts +17 -0
- package/dist/runtime/stageStreamLog.js +89 -0
- package/dist/runtime/stageWorker.js +2 -1
- package/dist/server/bootstrap.d.ts +2 -0
- package/dist/server/bootstrap.js +22 -4
- package/dist/server/createHttpHost.js +6 -0
- package/dist/server/ensureGlobalService.d.ts +35 -0
- package/dist/server/ensureGlobalService.js +138 -0
- package/dist/server/http.d.ts +20 -1
- package/dist/server/http.js +450 -402
- package/dist/server/mcpHost.js +7 -8
- package/dist/server/operatorResults.d.ts +10 -0
- package/dist/server/operatorResults.js +24 -0
- package/dist/server/providerRoutes.d.ts +0 -6
- package/dist/server/providerRoutes.js +10 -18
- package/dist/types/pipeline.d.ts +8 -0
- package/dist/ui/assets/{index-DMVdl1qB.js → index-Da5EHU6M.js} +5 -5
- package/dist/ui/index.html +1 -1
- package/package.json +2 -1
- package/skills/stageflow/references/control-surface.md +2 -2
- package/skills/stageflow-run/SKILL.md +27 -9
- package/skills/stageflow-run/references/mcp-call.md +1 -1
- package/skills/stageflow-run/scripts/mcp-call.mjs +9 -0
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { RunSubmissionExistsError } from "../runstore/submission.js";
|
|
2
3
|
import { readFile, realpath } from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
5
|
+
import { findProjectRoot } from "../project/findProjectRoot.js";
|
|
4
6
|
import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
|
|
5
7
|
import { loadRunContext } from "./resumeReconstruct.js";
|
|
6
8
|
import { assertTimedOutStageEligible, reconstructTimedOutAndContinue, } from "./resumeTimedOut.js";
|
|
@@ -12,7 +14,7 @@ import { resolveFeedbackLoopDecision, } from "./feedbackLoopDecision.js";
|
|
|
12
14
|
import { RunRetryCoordinator, readRetryRootWaitTimeoutMs, } from "./runRetryCoordinator.js";
|
|
13
15
|
import { readMaxActiveStageProcesses, readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
|
|
14
16
|
import { StageProcessLauncher } from "./stageProcessLauncher.js";
|
|
15
|
-
import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount,
|
|
17
|
+
import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, readMaxConcurrentFromGlobal, writeMaxConcurrentToGlobal, } from "./settingsFile.js";
|
|
16
18
|
import { StageHitlController, waitKey, } from "./stageHitl.js";
|
|
17
19
|
import { resolveAndValidateCheckout, } from "./stageRoots.js";
|
|
18
20
|
import { orchestrateAnswerResume } from "./answerResume.js";
|
|
@@ -21,6 +23,12 @@ import { resumeSessionFilePath } from "./stageAttemptContext.js";
|
|
|
21
23
|
import { resolveStartTaskInput } from "./taskInput.js";
|
|
22
24
|
import { failStageAsInterrupted, syncRunStatusFromStages, } from "./stageRecovery.js";
|
|
23
25
|
import { blocksGenericRetry, manualRecoveryEligibility, readManualRecoveryState, } from "./manualRecoveryState.js";
|
|
26
|
+
function existingSubmissionResult(existing, request) {
|
|
27
|
+
if (existing.requestHash !== request.requestHash) {
|
|
28
|
+
return { ok: false, status: 409, reason: "Submission key was already used for different input" };
|
|
29
|
+
}
|
|
30
|
+
return { ok: true, runId: existing.runId, reused: true };
|
|
31
|
+
}
|
|
24
32
|
const DEFAULT_MAX_CONCURRENT = 3;
|
|
25
33
|
const STARTUP_RECONCILE_REASON = "process_interrupted: no active worker (server restart)";
|
|
26
34
|
const OPERATOR_ABANDON_REASON = "process_interrupted: operator abandoned stage";
|
|
@@ -44,6 +52,7 @@ async function toCheckoutLeaseKey(absPath) {
|
|
|
44
52
|
}
|
|
45
53
|
export class RunManager {
|
|
46
54
|
options;
|
|
55
|
+
submissionsInFlight = new Map();
|
|
47
56
|
active = new Map();
|
|
48
57
|
checkoutLeases = new Map();
|
|
49
58
|
provisionalIds = new Set();
|
|
@@ -94,10 +103,9 @@ export class RunManager {
|
|
|
94
103
|
this.cwd = options.cwd ?? process.cwd();
|
|
95
104
|
this.projectRoot = options.projectRoot ?? this.cwd;
|
|
96
105
|
this.isGitProject = options.isGitProject ?? false;
|
|
97
|
-
const settingsCtx = projectSettingsContext(this.cwd, this.projectRoot, this.isGitProject);
|
|
98
106
|
this.maxConcurrent =
|
|
99
107
|
options.maxConcurrent ??
|
|
100
|
-
|
|
108
|
+
readMaxConcurrentFromGlobal() ??
|
|
101
109
|
parseMaxConcurrent(process.env.STAGEFLOW_MAX_CONCURRENT_RUNS);
|
|
102
110
|
this.maxActiveStagesPerRun = readMaxActiveStagesPerRun(process.env, options.maxActiveStagesPerRun);
|
|
103
111
|
this.executionMode = readStageExecutionMode(process.env, options.executionMode);
|
|
@@ -119,7 +127,7 @@ export class RunManager {
|
|
|
119
127
|
throw new Error(INVALID_SLOT_COUNT_MESSAGE);
|
|
120
128
|
}
|
|
121
129
|
this.maxConcurrent = parsed;
|
|
122
|
-
|
|
130
|
+
writeMaxConcurrentToGlobal(parsed);
|
|
123
131
|
return this.getHealth();
|
|
124
132
|
}
|
|
125
133
|
getActiveCount() {
|
|
@@ -348,8 +356,48 @@ export class RunManager {
|
|
|
348
356
|
}
|
|
349
357
|
return { ok: true, runId, stageId };
|
|
350
358
|
}
|
|
351
|
-
async
|
|
359
|
+
async startRunOnce(input, submission) {
|
|
360
|
+
if (!submission.key.trim() || submission.key.length > 256 || !/^[a-f0-9]{64}$/.test(submission.requestHash)) {
|
|
361
|
+
return { ok: false, status: 400, reason: "Submission requires a key and SHA-256 request hash" };
|
|
362
|
+
}
|
|
363
|
+
const pending = this.submissionsInFlight.get(submission.key);
|
|
364
|
+
if (pending) {
|
|
365
|
+
if (pending.requestHash !== submission.requestHash) {
|
|
366
|
+
return { ok: false, status: 409, reason: "Submission key was already used for different input" };
|
|
367
|
+
}
|
|
368
|
+
const result = await pending.result;
|
|
369
|
+
return result.ok ? { ...result, reused: true } : result;
|
|
370
|
+
}
|
|
371
|
+
const result = this.startSubmittedRun(input, submission);
|
|
372
|
+
this.submissionsInFlight.set(submission.key, { requestHash: submission.requestHash, result });
|
|
373
|
+
try {
|
|
374
|
+
return await result;
|
|
375
|
+
}
|
|
376
|
+
finally {
|
|
377
|
+
this.submissionsInFlight.delete(submission.key);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
async startSubmittedRun(input, submission) {
|
|
381
|
+
const existing = await this.options.store.getRunBySubmission(submission.key);
|
|
382
|
+
if (existing)
|
|
383
|
+
return existingSubmissionResult(existing, submission);
|
|
384
|
+
try {
|
|
385
|
+
const result = await this.startRun(input, submission);
|
|
386
|
+
return result.ok ? { ok: true, runId: result.runId, reused: false, done: result.done } : result;
|
|
387
|
+
}
|
|
388
|
+
catch (error) {
|
|
389
|
+
if (error instanceof RunSubmissionExistsError)
|
|
390
|
+
return existingSubmissionResult(error.submission, submission);
|
|
391
|
+
throw error;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
async startRun(input, submission) {
|
|
352
395
|
const cwd = this.options.cwd ?? process.cwd();
|
|
396
|
+
// An inline pipeline has no filesystem anchor to derive a project root from.
|
|
397
|
+
const derivedProjectRoot = typeof input.pipeline === "string"
|
|
398
|
+
? (findProjectRoot(path.dirname(path.resolve(cwd, input.pipeline))) ??
|
|
399
|
+
this.projectRoot)
|
|
400
|
+
: this.projectRoot;
|
|
353
401
|
let resolved;
|
|
354
402
|
try {
|
|
355
403
|
resolved = resolveStartTaskInput(input, cwd);
|
|
@@ -384,7 +432,7 @@ export class RunManager {
|
|
|
384
432
|
gitSha: input.gitSha,
|
|
385
433
|
ciPrUrl: input.ciPrUrl,
|
|
386
434
|
ciJobUrl: input.ciJobUrl,
|
|
387
|
-
},
|
|
435
|
+
}, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined, submission);
|
|
388
436
|
}
|
|
389
437
|
async rerun(runId) {
|
|
390
438
|
const cwd = this.options.cwd ?? process.cwd();
|
|
@@ -456,6 +504,7 @@ export class RunManager {
|
|
|
456
504
|
}
|
|
457
505
|
const insertedForResume = !wasActive;
|
|
458
506
|
try {
|
|
507
|
+
const runProjectRoot = detail.project_root ?? this.projectRoot;
|
|
459
508
|
const done = reconstructTimedOutAndContinue({
|
|
460
509
|
runId,
|
|
461
510
|
stageId,
|
|
@@ -463,9 +512,9 @@ export class RunManager {
|
|
|
463
512
|
store: this.options.store,
|
|
464
513
|
hitl: this.hitl,
|
|
465
514
|
executionMode: this.executionMode,
|
|
466
|
-
cwd:
|
|
515
|
+
cwd: runProjectRoot,
|
|
467
516
|
maxActiveStagesPerRun: this.maxActiveStagesPerRun,
|
|
468
|
-
factoryCwd:
|
|
517
|
+
factoryCwd: runProjectRoot,
|
|
469
518
|
...(this.stageProcessLauncher !== undefined
|
|
470
519
|
? { stageProcessLauncher: this.stageProcessLauncher }
|
|
471
520
|
: {}),
|
|
@@ -585,6 +634,7 @@ export class RunManager {
|
|
|
585
634
|
};
|
|
586
635
|
}
|
|
587
636
|
try {
|
|
637
|
+
let meta;
|
|
588
638
|
let orchestrationConflict = this.active.has(runId) &&
|
|
589
639
|
!this.attachedWaiting.has(waitKey(runId, stageId));
|
|
590
640
|
if (!this.retryCoordinator.isActive(runId)) {
|
|
@@ -607,7 +657,7 @@ export class RunManager {
|
|
|
607
657
|
let detail;
|
|
608
658
|
try {
|
|
609
659
|
detail = await this.options.store.readRun(runId);
|
|
610
|
-
await this.options.store.readRunMeta(runId);
|
|
660
|
+
meta = await this.options.store.readRunMeta(runId);
|
|
611
661
|
}
|
|
612
662
|
catch {
|
|
613
663
|
this.retryInFlight.delete(retryKey);
|
|
@@ -626,12 +676,21 @@ export class RunManager {
|
|
|
626
676
|
}
|
|
627
677
|
}
|
|
628
678
|
}
|
|
679
|
+
if (meta === undefined) {
|
|
680
|
+
try {
|
|
681
|
+
meta = await this.options.store.readRunMeta(runId);
|
|
682
|
+
}
|
|
683
|
+
catch {
|
|
684
|
+
this.retryInFlight.delete(retryKey);
|
|
685
|
+
return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
|
|
686
|
+
}
|
|
687
|
+
}
|
|
629
688
|
const result = await this.retryCoordinator.retryStage({
|
|
630
689
|
runId,
|
|
631
690
|
stageId,
|
|
632
691
|
store: this.options.store,
|
|
633
692
|
agent: this.options.agent,
|
|
634
|
-
cwd: this.
|
|
693
|
+
cwd: meta.project_root ?? this.projectRoot,
|
|
635
694
|
operatorCatalog: this.options.operatorCatalog,
|
|
636
695
|
maxActiveStagesPerRun: this.maxActiveStagesPerRun,
|
|
637
696
|
executionMode: this.executionMode,
|
|
@@ -980,6 +1039,7 @@ export class RunManager {
|
|
|
980
1039
|
};
|
|
981
1040
|
}
|
|
982
1041
|
const { meta, task, loaded } = await loadRunContext(this.options.store, runId, this.cwd);
|
|
1042
|
+
const runProjectRoot = meta.project_root ?? this.projectRoot;
|
|
983
1043
|
const done = runPipelineDag({
|
|
984
1044
|
prepared: {
|
|
985
1045
|
task,
|
|
@@ -990,8 +1050,8 @@ export class RunManager {
|
|
|
990
1050
|
},
|
|
991
1051
|
agent: this.options.agent,
|
|
992
1052
|
store: this.options.store,
|
|
993
|
-
cwd:
|
|
994
|
-
projectRoot:
|
|
1053
|
+
cwd: runProjectRoot,
|
|
1054
|
+
projectRoot: runProjectRoot,
|
|
995
1055
|
checkoutRoot: meta.checkout_root,
|
|
996
1056
|
hitl: this.hitl,
|
|
997
1057
|
operatorCatalog: this.options.operatorCatalog,
|
|
@@ -1041,10 +1101,12 @@ export class RunManager {
|
|
|
1041
1101
|
const sessionFilePath = resumeSessionFilePath(workspaceDir, stageId, attempt);
|
|
1042
1102
|
const eventOptions = { attempt };
|
|
1043
1103
|
try {
|
|
1104
|
+
const runMeta = await store.readRunMeta(runId);
|
|
1105
|
+
const runProjectRoot = runMeta.project_root ?? this.projectRoot;
|
|
1044
1106
|
const launchResult = await launcher.launch({
|
|
1045
1107
|
runId,
|
|
1046
1108
|
stageId,
|
|
1047
|
-
rootDir:
|
|
1109
|
+
rootDir: runProjectRoot,
|
|
1048
1110
|
mode: "resume",
|
|
1049
1111
|
resumeAnswer: opaqueAnswer,
|
|
1050
1112
|
attempt,
|
|
@@ -1079,8 +1141,8 @@ export class RunManager {
|
|
|
1079
1141
|
run: { runId, workspaceDir: store.getWorkspaceDir(runId) },
|
|
1080
1142
|
agent: this.options.agent,
|
|
1081
1143
|
store,
|
|
1082
|
-
cwd: this.
|
|
1083
|
-
projectRoot: this.projectRoot,
|
|
1144
|
+
cwd: meta.project_root ?? this.projectRoot,
|
|
1145
|
+
projectRoot: meta.project_root ?? this.projectRoot,
|
|
1084
1146
|
checkoutRoot: meta.checkout_root,
|
|
1085
1147
|
hitl: this.hitl,
|
|
1086
1148
|
operatorCatalog: this.options.operatorCatalog,
|
|
@@ -1137,7 +1199,7 @@ export class RunManager {
|
|
|
1137
1199
|
this.attachedWaiting.delete(`${runId}\0${stageId}`);
|
|
1138
1200
|
}
|
|
1139
1201
|
}
|
|
1140
|
-
async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath) {
|
|
1202
|
+
async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath, submission) {
|
|
1141
1203
|
let checkoutKey;
|
|
1142
1204
|
try {
|
|
1143
1205
|
const task = loadTaskFromYaml(taskYaml, taskLabel);
|
|
@@ -1159,6 +1221,7 @@ export class RunManager {
|
|
|
1159
1221
|
return reserved.failure;
|
|
1160
1222
|
try {
|
|
1161
1223
|
const started = await startPipeline({
|
|
1224
|
+
submission,
|
|
1162
1225
|
agent: this.options.agent,
|
|
1163
1226
|
store: this.options.store,
|
|
1164
1227
|
taskYaml,
|
|
@@ -1182,7 +1245,7 @@ export class RunManager {
|
|
|
1182
1245
|
}
|
|
1183
1246
|
catch (err) {
|
|
1184
1247
|
this.clearReservation(reserved.provisionalId);
|
|
1185
|
-
if (err instanceof PipelineValidationError) {
|
|
1248
|
+
if (err instanceof PipelineValidationError || err instanceof RunSubmissionExistsError) {
|
|
1186
1249
|
throw err;
|
|
1187
1250
|
}
|
|
1188
1251
|
return {
|
|
@@ -1277,6 +1340,7 @@ export class RunManager {
|
|
|
1277
1340
|
return false;
|
|
1278
1341
|
}
|
|
1279
1342
|
try {
|
|
1343
|
+
const runProjectRoot = meta.project_root ?? this.projectRoot;
|
|
1280
1344
|
const { meta: loadedMeta, task, loaded, workspaceDir } = await loadRunContext(this.options.store, runId, this.cwd);
|
|
1281
1345
|
const promise = runPipelineDag({
|
|
1282
1346
|
prepared: {
|
|
@@ -1285,8 +1349,8 @@ export class RunManager {
|
|
|
1285
1349
|
run: { runId, workspaceDir },
|
|
1286
1350
|
agent: this.options.agent,
|
|
1287
1351
|
store: this.options.store,
|
|
1288
|
-
cwd:
|
|
1289
|
-
projectRoot:
|
|
1352
|
+
cwd: runProjectRoot,
|
|
1353
|
+
projectRoot: runProjectRoot,
|
|
1290
1354
|
checkoutRoot: loadedMeta.checkout_root,
|
|
1291
1355
|
hitl: this.hitl,
|
|
1292
1356
|
operatorCatalog: this.options.operatorCatalog,
|
|
@@ -23,6 +23,8 @@ export declare function readMaxConcurrentFromContext(ctx: ProjectContext): numbe
|
|
|
23
23
|
export declare function readMaxConcurrentFromFile(cwd: string): number | undefined;
|
|
24
24
|
export declare function writeMaxConcurrentToContext(ctx: ProjectContext, maxConcurrent: number): void;
|
|
25
25
|
export declare function writeMaxConcurrentToFile(cwd: string, maxConcurrent: number): void;
|
|
26
|
+
export declare function readMaxConcurrentFromGlobal(): number | undefined;
|
|
27
|
+
export declare function writeMaxConcurrentToGlobal(maxConcurrent: number): void;
|
|
26
28
|
export declare function readCredentialSourceFromFile(cwd: string): CredentialSource | undefined;
|
|
27
29
|
export declare function writeCredentialSourceToContext(ctx: ProjectContext, credentialSource: CredentialSource): void;
|
|
28
30
|
export declare function writeCredentialSourceToFile(cwd: string, credentialSource: CredentialSource): void;
|
|
@@ -150,6 +150,25 @@ export function writeMaxConcurrentToContext(ctx, maxConcurrent) {
|
|
|
150
150
|
export function writeMaxConcurrentToFile(cwd, maxConcurrent) {
|
|
151
151
|
writeFactorySettings(cwd, { maxConcurrent });
|
|
152
152
|
}
|
|
153
|
+
export function readMaxConcurrentFromGlobal() {
|
|
154
|
+
return parseFactorySettingsFromRaw(readRawSettingsFromFile(globalSettingsFilePath())).maxConcurrent;
|
|
155
|
+
}
|
|
156
|
+
export function writeMaxConcurrentToGlobal(maxConcurrent) {
|
|
157
|
+
if (parseSlotCount(maxConcurrent) === undefined) {
|
|
158
|
+
throw new Error(INVALID_SLOT_COUNT_MESSAGE);
|
|
159
|
+
}
|
|
160
|
+
const current = parseFactorySettingsFromRaw(readRawSettingsFromFile(globalSettingsFilePath()));
|
|
161
|
+
const next = { ...current, maxConcurrent };
|
|
162
|
+
const out = {};
|
|
163
|
+
if (next.maxConcurrent !== undefined) {
|
|
164
|
+
out.maxConcurrent = next.maxConcurrent;
|
|
165
|
+
}
|
|
166
|
+
if (next.credentialSource !== undefined) {
|
|
167
|
+
out.credentialSource = next.credentialSource;
|
|
168
|
+
}
|
|
169
|
+
mkdirSync(path.dirname(globalSettingsFilePath()), { recursive: true });
|
|
170
|
+
writeFileSync(globalSettingsFilePath(), `${JSON.stringify(out, null, 2)}\n`, "utf8");
|
|
171
|
+
}
|
|
153
172
|
export function readCredentialSourceFromFile(cwd) {
|
|
154
173
|
return readFactorySettings(cwd).credentialSource;
|
|
155
174
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { AgentPort, FeedbackLoopContext, StageHandle, StageRunInput, StageSessionMode } from "../agent/port.js";
|
|
2
|
+
import { type StageStreamLogWriter } from "./stageStreamLog.js";
|
|
2
3
|
import type { RunStore } from "../runstore/port.js";
|
|
3
4
|
import type { StageEnvelope } from "../types/envelope.js";
|
|
4
5
|
import type { ResolvedPipelineDag } from "../types/pipeline.js";
|
|
@@ -23,6 +24,8 @@ export type StageAttemptOpenInput = {
|
|
|
23
24
|
attemptCtx?: StageAttemptContext;
|
|
24
25
|
operatorCatalog?: OperatorCatalog;
|
|
25
26
|
onActivity?: StageRunInput["onActivity"];
|
|
27
|
+
/** Test seam; defaults to the real createStageStreamLogWriter. */
|
|
28
|
+
streamLogWriterFactory?: (streamLogPath: string) => StageStreamLogWriter;
|
|
26
29
|
roots?: StageRoots;
|
|
27
30
|
resumeToken?: string;
|
|
28
31
|
sessionMode?: StageSessionMode;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { resolveSkillByName } from "../config/listSkills.js";
|
|
2
2
|
import { mkdir } from "node:fs/promises";
|
|
3
3
|
import { MCP_CATALOG_FILENAME, STAGEFLOW_STAGE_ARTIFACTS_DIR_ENV, StageMcpError, resolveStageMcpServers, stampStagePromptArtifactsDir, } from "../config/resolveStageMcpServers.js";
|
|
4
|
-
import { attemptArtifactsDir } from "../runstore/workspaceLayout.js";
|
|
4
|
+
import { attemptArtifactsDir, attemptStreamLogPath } from "../runstore/workspaceLayout.js";
|
|
5
|
+
import { createStageStreamLogWriter, } from "./stageStreamLog.js";
|
|
5
6
|
import { resolveCloneEmitContext, resolveForkEmitContext } from "../config/resolveForkEmitContext.js";
|
|
6
7
|
import { createAttemptQaTrailReader } from "../hitl/qaTrail.js";
|
|
7
8
|
import { assertPriorInputPayload, payloadInstanceMismatch, } from "../envelope/payloadSchema.js";
|
|
@@ -199,6 +200,7 @@ export async function openStageAttempt(input) {
|
|
|
199
200
|
return priorInput;
|
|
200
201
|
const roots = resolveAttemptRoots(input, stageId);
|
|
201
202
|
const attempt = input.attemptCtx?.attempt ?? 1;
|
|
203
|
+
const streamWriter = (input.streamLogWriterFactory ?? createStageStreamLogWriter)(attemptStreamLogPath(input.workspaceDir, stageId, attempt));
|
|
202
204
|
const resumeToken = input.resumeToken ??
|
|
203
205
|
resumeSessionFilePath(input.workspaceDir, stageId, attempt);
|
|
204
206
|
const forkEmitContext = resolveForkEmitContext(input.dag, definitionId);
|
|
@@ -231,7 +233,11 @@ export async function openStageAttempt(input) {
|
|
|
231
233
|
: {}),
|
|
232
234
|
resumeToken,
|
|
233
235
|
...(input.sessionMode !== undefined ? { sessionMode: input.sessionMode } : {}),
|
|
234
|
-
onActivity:
|
|
236
|
+
onActivity: (event) => {
|
|
237
|
+
input.onActivity?.(event);
|
|
238
|
+
void streamWriter.flush();
|
|
239
|
+
},
|
|
240
|
+
onAssistantTextDelta: streamWriter.onDelta,
|
|
235
241
|
forkEmitContext,
|
|
236
242
|
cloneEmitContext,
|
|
237
243
|
...(feedbackLoopEmitContext !== undefined ? { feedbackLoopEmitContext } : {}),
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** First line of every stream.log: the logical byte offset of the content that follows. */
|
|
2
|
+
export declare const STREAM_LOG_HEADER_PREFIX = "BASE:";
|
|
3
|
+
export declare function encodeStreamLogHeader(base: number): string;
|
|
4
|
+
/** Split a stream.log buffer into its logical base offset and content bytes. */
|
|
5
|
+
export declare function parseStreamLogHeader(raw: Buffer): {
|
|
6
|
+
base: number;
|
|
7
|
+
headerLength: number;
|
|
8
|
+
};
|
|
9
|
+
export type StageStreamLogWriter = {
|
|
10
|
+
onDelta(delta: string): void;
|
|
11
|
+
/** Idempotent; resolves once any pending buffered text has been written to disk. */
|
|
12
|
+
flush(): Promise<void>;
|
|
13
|
+
};
|
|
14
|
+
export declare function createStageStreamLogWriter(streamLogPath: string, options?: {
|
|
15
|
+
flushThrottleMs?: number;
|
|
16
|
+
capBytes?: number;
|
|
17
|
+
}): StageStreamLogWriter;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { STREAM_FLUSH_THROTTLE_MS } from "../agent/activityObserver.js";
|
|
4
|
+
import { redactSecrets } from "../agent/streamLogRedact.js";
|
|
5
|
+
const DEFAULT_CAP_BYTES = 256 * 1024;
|
|
6
|
+
/** First line of every stream.log: the logical byte offset of the content that follows. */
|
|
7
|
+
export const STREAM_LOG_HEADER_PREFIX = "BASE:";
|
|
8
|
+
export function encodeStreamLogHeader(base) {
|
|
9
|
+
return `${STREAM_LOG_HEADER_PREFIX}${base}\n`;
|
|
10
|
+
}
|
|
11
|
+
/** Split a stream.log buffer into its logical base offset and content bytes. */
|
|
12
|
+
export function parseStreamLogHeader(raw) {
|
|
13
|
+
const prefix = Buffer.from(STREAM_LOG_HEADER_PREFIX, "utf8");
|
|
14
|
+
if (raw.subarray(0, prefix.length).equals(prefix)) {
|
|
15
|
+
const newlineIdx = raw.indexOf(0x0a);
|
|
16
|
+
if (newlineIdx !== -1) {
|
|
17
|
+
const baseStr = raw.subarray(prefix.length, newlineIdx).toString("utf8");
|
|
18
|
+
const base = Number(baseStr);
|
|
19
|
+
if (Number.isFinite(base) && base >= 0) {
|
|
20
|
+
return { base, headerLength: newlineIdx + 1 };
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return { base: 0, headerLength: 0 };
|
|
25
|
+
}
|
|
26
|
+
async function appendChunk(streamLogPath, chunk, capBytes) {
|
|
27
|
+
await mkdir(path.dirname(streamLogPath), { recursive: true });
|
|
28
|
+
let base = 0;
|
|
29
|
+
let contentBytes;
|
|
30
|
+
try {
|
|
31
|
+
const raw = await readFile(streamLogPath);
|
|
32
|
+
const parsed = parseStreamLogHeader(raw);
|
|
33
|
+
base = parsed.base;
|
|
34
|
+
contentBytes = raw.subarray(parsed.headerLength);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
contentBytes = Buffer.alloc(0);
|
|
38
|
+
}
|
|
39
|
+
contentBytes = Buffer.concat([contentBytes, Buffer.from(chunk, "utf8")]);
|
|
40
|
+
const headerBytes = Buffer.from(encodeStreamLogHeader(base), "utf8");
|
|
41
|
+
if (headerBytes.length + contentBytes.length > capBytes) {
|
|
42
|
+
let dropBytes = Math.floor(contentBytes.length / 2);
|
|
43
|
+
// Never split a multi-byte UTF-8 codepoint: skip past continuation bytes.
|
|
44
|
+
while (dropBytes < contentBytes.length &&
|
|
45
|
+
(contentBytes[dropBytes] & 0xc0) === 0x80) {
|
|
46
|
+
dropBytes++;
|
|
47
|
+
}
|
|
48
|
+
base += dropBytes;
|
|
49
|
+
contentBytes = contentBytes.subarray(dropBytes);
|
|
50
|
+
}
|
|
51
|
+
const finalHeader = Buffer.from(encodeStreamLogHeader(base), "utf8");
|
|
52
|
+
await writeFile(streamLogPath, Buffer.concat([finalHeader, contentBytes]));
|
|
53
|
+
}
|
|
54
|
+
export function createStageStreamLogWriter(streamLogPath, options = {}) {
|
|
55
|
+
const flushThrottleMs = options.flushThrottleMs ?? STREAM_FLUSH_THROTTLE_MS;
|
|
56
|
+
const capBytes = options.capBytes ?? DEFAULT_CAP_BYTES;
|
|
57
|
+
let buffer = "";
|
|
58
|
+
let timer = null;
|
|
59
|
+
// Serializes disk writes so the throttle timer and an eager flush() never race.
|
|
60
|
+
let writeChain = Promise.resolve();
|
|
61
|
+
function flushNow() {
|
|
62
|
+
if (timer !== null) {
|
|
63
|
+
clearTimeout(timer);
|
|
64
|
+
timer = null;
|
|
65
|
+
}
|
|
66
|
+
if (!buffer)
|
|
67
|
+
return writeChain;
|
|
68
|
+
const chunk = redactSecrets(buffer);
|
|
69
|
+
buffer = "";
|
|
70
|
+
writeChain = writeChain.then(() => appendChunk(streamLogPath, chunk, capBytes));
|
|
71
|
+
return writeChain;
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
onDelta(delta) {
|
|
75
|
+
if (!delta)
|
|
76
|
+
return;
|
|
77
|
+
buffer += delta;
|
|
78
|
+
if (timer === null) {
|
|
79
|
+
timer = setTimeout(() => {
|
|
80
|
+
timer = null;
|
|
81
|
+
void flushNow();
|
|
82
|
+
}, flushThrottleMs);
|
|
83
|
+
}
|
|
84
|
+
},
|
|
85
|
+
flush() {
|
|
86
|
+
return flushNow();
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
}
|
|
@@ -3,6 +3,7 @@ import { asAgentBackendId } from "../agent/agentBackend.js";
|
|
|
3
3
|
import { loadStageflowManifestOutcome } from "../config/loadStageflowManifest.js";
|
|
4
4
|
import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
|
|
5
5
|
import { createRunStore } from "../runstore/createStore.js";
|
|
6
|
+
import { globalStageflowHome } from "../project/globalHome.js";
|
|
6
7
|
import { loadRunContext } from "./resumeReconstruct.js";
|
|
7
8
|
import { bindPiAgentDirEnv, rootsForStageWorker, withResolvedAuthPath, } from "./stageRoots.js";
|
|
8
9
|
import { runStage, isRunStageWaiting, } from "./stageRunner.js";
|
|
@@ -11,7 +12,7 @@ import { attemptContext } from "./stageAttemptContext.js";
|
|
|
11
12
|
import { loadActiveFeedbackLoopContext } from "./feedbackLoopCoordinator.js";
|
|
12
13
|
import { outcomeToWorkerResult, STAGE_WORKER_EXIT, } from "./stageWorkerProtocol.js";
|
|
13
14
|
export async function runStageWorker(input) {
|
|
14
|
-
const store = createRunStore({ rootDir:
|
|
15
|
+
const store = createRunStore({ rootDir: globalStageflowHome() });
|
|
15
16
|
const { meta, task, loaded } = await loadRunContext(store, input.runId, input.rootDir);
|
|
16
17
|
const definitionId = definitionIdForInstance(meta.pipeline_dag, input.stageId);
|
|
17
18
|
const stage = loaded.stages.find((s) => s.id === definitionId);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type A2aHost } from "../a2a/server.js";
|
|
1
2
|
import type { AgentPort } from "../agent/port.js";
|
|
2
3
|
import type { ProviderAuthContext } from "../agent/providerAuth.js";
|
|
3
4
|
import { type RunStoreKind } from "../runstore/createStore.js";
|
|
@@ -18,6 +19,7 @@ export type StageflowHostOptions = {
|
|
|
18
19
|
runChangeBus?: RunChangeBus;
|
|
19
20
|
};
|
|
20
21
|
export type StageflowHostBootstrap = {
|
|
22
|
+
a2a?: A2aHost;
|
|
21
23
|
cwd: string;
|
|
22
24
|
agentDir: string;
|
|
23
25
|
rootDir: string;
|
package/dist/server/bootstrap.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import {
|
|
2
|
+
import { createA2aHost } from "../a2a/server.js";
|
|
3
|
+
import { createRunStoreWithConnection } from "../runstore/createStore.js";
|
|
4
|
+
import { resolveA2aConfigPath } from "../a2a/configDiscovery.js";
|
|
3
5
|
import { resolveStageflowContext } from "../project/resolveStageflowContext.js";
|
|
4
6
|
import { findProjectRoot } from "../project/findProjectRoot.js";
|
|
5
7
|
import { RunManager } from "../runtime/runManager.js";
|
|
@@ -14,8 +16,16 @@ export async function bootstrapStageflowHost(options) {
|
|
|
14
16
|
const isGitProject = options.rootDir !== undefined
|
|
15
17
|
? findProjectRoot(rootDir) !== null
|
|
16
18
|
: ctx.isGitProject;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
+
let rawStore;
|
|
20
|
+
let sqliteConnection;
|
|
21
|
+
if (options.store) {
|
|
22
|
+
rawStore = options.store;
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
const created = createRunStoreWithConnection({ rootDir: ctx.globalHome, kind: options.storeKind });
|
|
26
|
+
rawStore = created.store;
|
|
27
|
+
sqliteConnection = created.connection;
|
|
28
|
+
}
|
|
19
29
|
const boundBus = isRunStoreWrapped(rawStore)
|
|
20
30
|
? getRunChangeBusFromWrappedStore(rawStore)
|
|
21
31
|
: undefined;
|
|
@@ -41,8 +51,16 @@ export async function bootstrapStageflowHost(options) {
|
|
|
41
51
|
const mcpStateless = resolveMcpStateless({
|
|
42
52
|
mcpStateless: options.mcpStateless,
|
|
43
53
|
});
|
|
44
|
-
const mcpHandler = createMcpHttpHandler({
|
|
54
|
+
const mcpHandler = createMcpHttpHandler({
|
|
55
|
+
manager,
|
|
56
|
+
store,
|
|
57
|
+
cwd,
|
|
58
|
+
runChangeBus,
|
|
59
|
+
providerAuthContext: options.providerAuthContext,
|
|
60
|
+
projectRoot: rootDir,
|
|
61
|
+
}, { mcpStateless });
|
|
45
62
|
return {
|
|
63
|
+
a2a: await createA2aHost({ manager, runStore: store, rootDir, connection: sqliteConnection }, resolveA2aConfigPath(rootDir)),
|
|
46
64
|
cwd,
|
|
47
65
|
agentDir,
|
|
48
66
|
rootDir,
|
|
@@ -17,6 +17,12 @@ export async function createHttpHost(options) {
|
|
|
17
17
|
const method = req.method ?? "GET";
|
|
18
18
|
const url = new URL(req.url ?? "/", `http://${host}:${port}`);
|
|
19
19
|
const pathname = url.pathname;
|
|
20
|
+
if (pathname === "/api/a2a/status" && method === "GET") {
|
|
21
|
+
json(res, 200, boot.a2a?.status ?? { state: "disabled" });
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
if (await boot.a2a?.handle(req, res, pathname))
|
|
25
|
+
return;
|
|
20
26
|
if (pathname === "/mcp") {
|
|
21
27
|
if (!validateMcpHost(req, res) || !validateMcpOrigin(req, res)) {
|
|
22
28
|
return;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { type ChildProcess, spawn as spawnDefault } from "node:child_process";
|
|
2
|
+
/** Coarse up/down probe result, used by CLI code that only needs "is it safe to talk to the host." */
|
|
3
|
+
export type HostProbeResult = "up" | "down";
|
|
4
|
+
/**
|
|
5
|
+
* Finer-grained probe result used to auto-start: "unreachable" (nothing
|
|
6
|
+
* listening, safe to spawn) is distinct from "unhealthy" (something is
|
|
7
|
+
* listening on the port but didn't answer with valid health JSON, so it's
|
|
8
|
+
* almost certainly not our service and spawning would just collide).
|
|
9
|
+
*/
|
|
10
|
+
export type ServiceProbeResult = "up" | "unreachable" | "unhealthy";
|
|
11
|
+
export declare function hostBaseUrl(): string;
|
|
12
|
+
export declare function probeGlobalServiceDetailed(): Promise<ServiceProbeResult>;
|
|
13
|
+
export declare function defaultProbeHost(): Promise<HostProbeResult>;
|
|
14
|
+
export type EnsureGlobalServiceResult = {
|
|
15
|
+
ok: true;
|
|
16
|
+
alreadyRunning: boolean;
|
|
17
|
+
} | {
|
|
18
|
+
ok: false;
|
|
19
|
+
reason: "port_occupied" | "spawn_failed" | "timed_out";
|
|
20
|
+
message: string;
|
|
21
|
+
};
|
|
22
|
+
export interface EnsureGlobalServiceOptions {
|
|
23
|
+
/** Path to the CLI entry script to spawn `<entry> mcp` against. Defaults to the running process's own entry (process.argv[1]). */
|
|
24
|
+
cliEntry?: string;
|
|
25
|
+
probeHost?: () => Promise<ServiceProbeResult>;
|
|
26
|
+
spawnFn?: (command: string, args: string[], options: Parameters<typeof spawnDefault>[2]) => ChildProcess;
|
|
27
|
+
pollIntervalMs?: number;
|
|
28
|
+
timeoutMs?: number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Health-probe-then-spawn: makes sure a global Stageflow service is
|
|
32
|
+
* reachable at the well-known port, starting one headlessly (`<cliEntry>
|
|
33
|
+
* mcp`, detached) if nothing answers yet. No-ops if one is already up.
|
|
34
|
+
*/
|
|
35
|
+
export declare function ensureGlobalService(options?: EnsureGlobalServiceOptions): Promise<EnsureGlobalServiceResult>;
|