stageflow 0.15.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -4
- package/dist/agent/activity.d.ts +3 -0
- package/dist/agent/claudeAdapter.js +84 -7
- package/dist/agent/claudeSession.d.ts +3 -0
- package/dist/agent/claudeTools.d.ts +1 -4
- package/dist/agent/claudeTools.js +1 -6
- package/dist/agent/fakeAgent.js +4 -5
- package/dist/agent/piAdapter.d.ts +2 -0
- package/dist/agent/piAdapter.js +30 -19
- package/dist/agent/port.d.ts +3 -0
- package/dist/cli/runOutput.js +32 -8
- package/dist/config/cloneChain.d.ts +12 -0
- package/dist/config/cloneChain.js +322 -0
- package/dist/config/createPipeline.js +101 -21
- package/dist/config/createStage.js +7 -0
- package/dist/config/loadOutcome.d.ts +1 -0
- package/dist/config/loadPipeline.d.ts +2 -0
- package/dist/config/loadPipeline.js +34 -22
- package/dist/config/loadStage.d.ts +2 -0
- package/dist/config/loadStage.js +135 -125
- package/dist/config/migrateYaml.js +14 -3
- package/dist/config/normalizePipelineStageEntry.d.ts +5 -9
- package/dist/config/normalizePipelineStageEntry.js +133 -60
- package/dist/config/parsePreEmitChecks.js +2 -2
- package/dist/config/pipelineNeeds.d.ts +15 -2
- package/dist/config/pipelineNeeds.js +86 -4
- package/dist/config/pipelineRoute.d.ts +46 -0
- package/dist/config/pipelineRoute.js +215 -0
- package/dist/config/pipelineStageKeys.d.ts +2 -0
- package/dist/config/pipelineStageKeys.js +15 -4
- package/dist/config/printTargetYaml.js +18 -12
- package/dist/config/resolveForkEmitContext.d.ts +1 -1
- package/dist/config/resolveForkEmitContext.js +3 -26
- package/dist/config/resolvePipelineDag.d.ts +1 -3
- package/dist/config/resolvePipelineDag.js +143 -186
- package/dist/config/routeIf.d.ts +24 -0
- package/dist/config/routeIf.js +247 -0
- package/dist/config/validateCatalog.d.ts +1 -1
- package/dist/config/validateCatalog.js +5 -3
- package/dist/config/yamlDialect.d.ts +5 -8
- package/dist/config/yamlDialect.js +37 -29
- package/dist/envelope/check.d.ts +0 -2
- package/dist/envelope/check.js +1 -57
- package/dist/envelope/feedbackLoop.js +0 -3
- package/dist/envelope/forkChoice.js +1 -1
- package/dist/envelope/payloadSchema.d.ts +14 -0
- package/dist/envelope/payloadSchema.js +87 -2
- package/dist/mcp/catalogTools.js +1 -2
- package/dist/package-meta.d.ts +1 -1
- package/dist/package-meta.js +1 -1
- package/dist/projection/projectRun.d.ts +0 -2
- package/dist/projection/projectRun.js +0 -3
- package/dist/prompt/priorEnvelope.js +1 -1
- package/dist/runstore/port.d.ts +10 -0
- package/dist/runstore/runProjection.js +9 -0
- package/dist/runstore/sqlite/SqliteRunStore.js +27 -4
- package/dist/runstore/sqlite/schema.d.ts +1 -1
- package/dist/runstore/sqlite/schema.js +2 -0
- package/dist/runstore/stageExecution.d.ts +3 -0
- package/dist/runstore/stageExecution.js +6 -2
- package/dist/runstore/stageInstanceId.d.ts +1 -0
- package/dist/runstore/stageInstanceId.js +9 -0
- package/dist/runstore/stageSnapshot.js +11 -0
- package/dist/runtime/cloneSchedule.d.ts +16 -33
- package/dist/runtime/cloneSchedule.js +155 -236
- package/dist/runtime/envelopeRouting.js +43 -94
- package/dist/runtime/forkGeneration.d.ts +2 -2
- package/dist/runtime/forkGeneration.js +29 -15
- package/dist/runtime/joinReadiness.d.ts +16 -0
- package/dist/runtime/joinReadiness.js +156 -0
- package/dist/runtime/pipelineScheduler.d.ts +0 -3
- package/dist/runtime/pipelineScheduler.js +81 -65
- package/dist/runtime/routeIfEval.d.ts +3 -0
- package/dist/runtime/routeIfEval.js +94 -0
- package/dist/runtime/runRetryCoordinator.js +0 -4
- package/dist/runtime/stageAttemptBootstrap.js +60 -11
- package/dist/runtime/stageRunner.js +2 -2
- package/dist/runtime/taskInput.js +1 -13
- package/dist/tools/emitStageEnvelope.d.ts +2 -3
- package/dist/tools/emitStageEnvelope.js +1 -65
- package/dist/types/envelope.d.ts +0 -2
- package/dist/types/forkChoice.d.ts +0 -16
- package/dist/types/pipeline.d.ts +86 -27
- package/dist/types/stage.d.ts +11 -7
- package/dist/types/usage.d.ts +16 -0
- package/dist/types/usage.js +24 -0
- package/dist/ui/assets/{index-uVHiae8s.css → index-DsgVjLUS.css} +1 -1
- package/dist/ui/assets/index-DwKGaQvQ.js +119 -0
- package/dist/ui/index.html +2 -2
- package/package.json +2 -1
- package/skills/stageflow-author/SKILL.md +4 -4
- package/skills/stageflow-author/assets/examples/branch-decision/hotfix.yaml +10 -2
- package/skills/stageflow-author/assets/examples/branch-decision/release-gate.pipeline.yaml +12 -4
- package/skills/stageflow-author/assets/examples/branch-decision/run-tests.yaml +6 -4
- package/skills/stageflow-author/assets/examples/branch-decision/ship.yaml +9 -1
- package/skills/stageflow-author/assets/examples/linear-review/draft.yaml +3 -0
- package/skills/stageflow-author/assets/examples/linear-review/publish.yaml +8 -0
- package/skills/stageflow-author/assets/examples/linear-review/review-loop.pipeline.yaml +5 -2
- package/skills/stageflow-author/assets/examples/linear-review/review.yaml +8 -0
- package/skills/stageflow-author/assets/examples/non-sdlc-digest/gather.yaml +3 -0
- package/skills/stageflow-author/assets/examples/non-sdlc-digest/research-digest.pipeline.yaml +5 -2
- package/skills/stageflow-author/assets/examples/non-sdlc-digest/send.yaml +8 -0
- package/skills/stageflow-author/assets/examples/non-sdlc-digest/summarize.yaml +8 -0
- package/skills/stageflow-author/references/catalog-mapping.md +159 -50
- package/skills/stageflow-author/references/catalog-write-conventions.md +3 -3
- package/skills/stageflow-author/references/stage-prompt-template.md +57 -17
- package/skills/stageflow-session-capture/SKILL.md +2 -2
- package/skills/stageflow-session-capture/assets/example-pipeline/example.pipeline.yaml +3 -1
- package/skills/stageflow-session-capture/assets/example-pipeline/implement.yaml +15 -0
- package/skills/stageflow-session-capture/assets/example-pipeline/research.yaml +3 -0
- package/skills/stageflow-session-capture/references/catalog-authoring.md +4 -2
- package/dist/envelope/cloneForks.d.ts +0 -2
- package/dist/envelope/cloneForks.js +0 -77
- package/dist/ui/assets/index-GtyiWiZg.js +0 -118
|
@@ -10,22 +10,35 @@ function asDagSnapshot(dag) {
|
|
|
10
10
|
stage_ids: dag.nodes.map((n) => n.id),
|
|
11
11
|
};
|
|
12
12
|
}
|
|
13
|
-
|
|
13
|
+
function nodeById(dag, id) {
|
|
14
|
+
return dag.nodes.find((n) => n.id === id);
|
|
15
|
+
}
|
|
16
|
+
function isCloneEmitter(node) {
|
|
17
|
+
return node?.clone_array_field !== undefined;
|
|
18
|
+
}
|
|
19
|
+
function isMintedCloneInstance(node) {
|
|
20
|
+
return (node !== undefined &&
|
|
21
|
+
node.definition_id !== undefined &&
|
|
22
|
+
node.definition_id !== node.id);
|
|
23
|
+
}
|
|
24
|
+
function parentIsCloneEmitter(dag, node) {
|
|
25
|
+
if (node?.needs === null || node?.needs === undefined)
|
|
26
|
+
return false;
|
|
27
|
+
return isCloneEmitter(nodeById(dag, node.needs));
|
|
28
|
+
}
|
|
29
|
+
/** Persistent route stages that fan out to Clone Chain children (or already have instances). */
|
|
14
30
|
export function routeRelatedForkParentIds(dag, routeStageIds) {
|
|
15
31
|
const routeSet = new Set(routeStageIds);
|
|
16
32
|
const parents = new Set();
|
|
17
33
|
const snapshot = asDagSnapshot(dag);
|
|
18
34
|
for (const stageId of routeStageIds) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
if (child?.clonable === true) {
|
|
22
|
-
parents.add(stageId);
|
|
23
|
-
}
|
|
35
|
+
if (isCloneEmitter(nodeById(dag, stageId))) {
|
|
36
|
+
parents.add(stageId);
|
|
24
37
|
}
|
|
25
38
|
}
|
|
26
39
|
for (const stageId of routeStageIds) {
|
|
27
|
-
const node = dag
|
|
28
|
-
if (node
|
|
40
|
+
const node = nodeById(dag, stageId);
|
|
41
|
+
if (node !== undefined && parentIsCloneEmitter(dag, node) && node.needs !== null) {
|
|
29
42
|
parents.add(node.needs);
|
|
30
43
|
}
|
|
31
44
|
}
|
|
@@ -38,24 +51,25 @@ export function routeRelatedForkParentIds(dag, routeStageIds) {
|
|
|
38
51
|
}
|
|
39
52
|
}
|
|
40
53
|
for (const stageId of routeStageIds) {
|
|
41
|
-
const node = dag
|
|
42
|
-
if (node
|
|
54
|
+
const node = nodeById(dag, stageId);
|
|
55
|
+
if (!parentIsCloneEmitter(dag, node) || isMintedCloneInstance(node))
|
|
43
56
|
continue;
|
|
44
57
|
const instances = instancesOfDefinition(snapshot, stageId).filter((id) => id !== stageId);
|
|
45
|
-
if (instances.length > 0 && node.needs !== null) {
|
|
58
|
+
if (instances.length > 0 && node !== undefined && node.needs !== null) {
|
|
46
59
|
parents.add(node.needs);
|
|
47
60
|
}
|
|
48
61
|
}
|
|
49
62
|
return [...parents];
|
|
50
63
|
}
|
|
51
|
-
/** Clone instance ids currently under
|
|
64
|
+
/** Clone instance ids currently under Clone Chain children of a fork parent. */
|
|
52
65
|
export function cloneInstanceIdsForForkParent(dag, forkParentStageId) {
|
|
53
66
|
const snapshot = asDagSnapshot(dag);
|
|
54
67
|
const ids = [];
|
|
55
68
|
const seen = new Set();
|
|
69
|
+
const parentIsEmitter = isCloneEmitter(nodeById(dag, forkParentStageId));
|
|
56
70
|
for (const childId of dag.childrenOf[forkParentStageId] ?? []) {
|
|
57
|
-
const child = dag
|
|
58
|
-
if (
|
|
71
|
+
const child = nodeById(dag, childId);
|
|
72
|
+
if (parentIsEmitter && !isMintedCloneInstance(child)) {
|
|
59
73
|
for (const id of instancesOfDefinition(snapshot, childId)) {
|
|
60
74
|
if (id === childId)
|
|
61
75
|
continue;
|
|
@@ -264,7 +278,7 @@ export function forkChoicePreservesReplaySource(dag, chosen, sourceStageId) {
|
|
|
264
278
|
return true;
|
|
265
279
|
}
|
|
266
280
|
const child = dag.nodes.find((n) => n.id === childId);
|
|
267
|
-
if (child
|
|
281
|
+
if (parentIsCloneEmitter(dag, child) && !isMintedCloneInstance(child)) {
|
|
268
282
|
const snapshot = asDagSnapshot(dag);
|
|
269
283
|
for (const instanceId of instancesOfDefinition(snapshot, childId)) {
|
|
270
284
|
if (instanceId === childId)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { StageEnvelope } from "../types/envelope.js";
|
|
2
|
+
import type { PipelineNeedEdge, ResolvedPipelineDag, ResolvedPipelineStageNode } from "../types/pipeline.js";
|
|
3
|
+
import type { StageScheduleState } from "./pipelineScheduler.js";
|
|
4
|
+
import { type RouteIfEval } from "./routeIfEval.js";
|
|
5
|
+
export declare function expandJoinParent(dag: ResolvedPipelineDag, catalogParentId: string): string[];
|
|
6
|
+
export declare function inboundEdgeFired(edge: PipelineNeedEdge, parentId: string, states: Map<string, StageScheduleState>, envelopes: Map<string, StageEnvelope>): boolean;
|
|
7
|
+
export declare function joinAllowsRun(dag: ResolvedPipelineDag, stageId: string, states: Map<string, StageScheduleState>, envelopes: Map<string, StageEnvelope>): boolean;
|
|
8
|
+
export type InboundAfterSuccess = RouteIfEval | "ungated";
|
|
9
|
+
export declare function classifyInboundAfterSuccess(node: Pick<ResolvedPipelineStageNode, "needs" | "needsEdges">, parentId: string, payload: Record<string, unknown> | undefined): InboundAfterSuccess;
|
|
10
|
+
export declare function isEagerSingleParentIfSkip(node: Pick<ResolvedPipelineStageNode, "needs" | "needsEdges">, parentId: string, payload: Record<string, unknown> | undefined): boolean;
|
|
11
|
+
/**
|
|
12
|
+
* Finds pending multi-parent joins that cannot run once every parent is
|
|
13
|
+
* terminal: all-skipped, or all-succeeded with an inbound `if` miss.
|
|
14
|
+
* A failed parent is not a stalled skip — that join stays pending.
|
|
15
|
+
*/
|
|
16
|
+
export declare function pickStalledJoinSkips(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes?: Map<string, StageEnvelope>): string[];
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { isNeedTerminalState, predecessorEdges } from "../config/pipelineNeeds.js";
|
|
2
|
+
import { instancesOfDefinition } from "../runstore/pipelineDagSnapshot.js";
|
|
3
|
+
import { evaluateRouteIf } from "./routeIfEval.js";
|
|
4
|
+
function definitionInstances(dag, catalogId) {
|
|
5
|
+
const snapshot = dag;
|
|
6
|
+
if (Array.isArray(snapshot.stage_ids)) {
|
|
7
|
+
return instancesOfDefinition(snapshot, catalogId);
|
|
8
|
+
}
|
|
9
|
+
return dag.nodes.some((n) => n.id === catalogId) ? [catalogId] : [];
|
|
10
|
+
}
|
|
11
|
+
export function expandJoinParent(dag, catalogParentId) {
|
|
12
|
+
const minted = definitionInstances(dag, catalogParentId).filter((id) => id !== catalogParentId);
|
|
13
|
+
return minted.length > 0 ? minted : [catalogParentId];
|
|
14
|
+
}
|
|
15
|
+
function expandJoinParents(dag, edges) {
|
|
16
|
+
const ids = [];
|
|
17
|
+
for (const edge of edges) {
|
|
18
|
+
ids.push(...expandJoinParent(dag, edge.id));
|
|
19
|
+
}
|
|
20
|
+
return ids;
|
|
21
|
+
}
|
|
22
|
+
function joinParentDisposition(dag, edges, states) {
|
|
23
|
+
const instances = expandJoinParents(dag, edges);
|
|
24
|
+
if (instances.length === 0)
|
|
25
|
+
return undefined;
|
|
26
|
+
let hasFailed = false;
|
|
27
|
+
let hasSucceeded = false;
|
|
28
|
+
let hasSkipped = false;
|
|
29
|
+
for (const id of instances) {
|
|
30
|
+
const state = states.get(id);
|
|
31
|
+
if (!isNeedTerminalState(state)) {
|
|
32
|
+
return { allTerminal: false };
|
|
33
|
+
}
|
|
34
|
+
if (state === "failed")
|
|
35
|
+
hasFailed = true;
|
|
36
|
+
if (state === "succeeded")
|
|
37
|
+
hasSucceeded = true;
|
|
38
|
+
if (state === "skipped")
|
|
39
|
+
hasSkipped = true;
|
|
40
|
+
}
|
|
41
|
+
return { allTerminal: true, hasFailed, hasSucceeded, hasSkipped };
|
|
42
|
+
}
|
|
43
|
+
export function inboundEdgeFired(edge, parentId, states, envelopes) {
|
|
44
|
+
if (edge.if === undefined)
|
|
45
|
+
return true;
|
|
46
|
+
if (states.get(parentId) !== "succeeded")
|
|
47
|
+
return false;
|
|
48
|
+
return evaluateRouteIf(edge.if, envelopes.get(parentId)?.payload) === "fire";
|
|
49
|
+
}
|
|
50
|
+
function inboundEdgeMissed(edge, parentId, envelopes) {
|
|
51
|
+
if (edge.if === undefined)
|
|
52
|
+
return false;
|
|
53
|
+
return evaluateRouteIf(edge.if, envelopes.get(parentId)?.payload) === "miss";
|
|
54
|
+
}
|
|
55
|
+
function everySucceededInboundEdgeFired(dag, edges, states, envelopes) {
|
|
56
|
+
for (const edge of edges) {
|
|
57
|
+
for (const id of expandJoinParent(dag, edge.id)) {
|
|
58
|
+
if (!inboundEdgeFired(edge, id, states, envelopes))
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
function anySucceededInboundEdgeMissed(dag, edges, envelopes) {
|
|
65
|
+
for (const edge of edges) {
|
|
66
|
+
for (const id of expandJoinParent(dag, edge.id)) {
|
|
67
|
+
if (inboundEdgeMissed(edge, id, envelopes))
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
function multiParentJoinAllowsRun(dag, node, states, envelopes) {
|
|
74
|
+
const edges = predecessorEdges(node);
|
|
75
|
+
if (expandJoinParents(dag, edges).length < 2)
|
|
76
|
+
return false;
|
|
77
|
+
const disposition = joinParentDisposition(dag, edges, states);
|
|
78
|
+
if (disposition === undefined || !disposition.allTerminal)
|
|
79
|
+
return false;
|
|
80
|
+
if (disposition.hasFailed)
|
|
81
|
+
return false;
|
|
82
|
+
if (!disposition.hasSucceeded)
|
|
83
|
+
return false;
|
|
84
|
+
if (!disposition.hasSkipped) {
|
|
85
|
+
return everySucceededInboundEdgeFired(dag, edges, states, envelopes);
|
|
86
|
+
}
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* A multi-parent join whose parents are all terminal, none failed, and
|
|
91
|
+
* either every parent skipped, or every parent succeeded but an inbound
|
|
92
|
+
* `if` missed. Those joins cannot run, so the scheduler force-skips them.
|
|
93
|
+
* A failed parent is not stalled: the join stays pending. A still-running
|
|
94
|
+
* parent is not stalled either.
|
|
95
|
+
*/
|
|
96
|
+
function isStalledMultiParentJoin(dag, node, states, envelopes) {
|
|
97
|
+
const edges = predecessorEdges(node);
|
|
98
|
+
if (edges.length <= 1)
|
|
99
|
+
return false;
|
|
100
|
+
const disposition = joinParentDisposition(dag, edges, states);
|
|
101
|
+
if (disposition === undefined || !disposition.allTerminal)
|
|
102
|
+
return false;
|
|
103
|
+
if (disposition.hasFailed)
|
|
104
|
+
return false;
|
|
105
|
+
if (!disposition.hasSucceeded)
|
|
106
|
+
return true;
|
|
107
|
+
if (disposition.hasSkipped)
|
|
108
|
+
return false;
|
|
109
|
+
return anySucceededInboundEdgeMissed(dag, edges, envelopes);
|
|
110
|
+
}
|
|
111
|
+
export function joinAllowsRun(dag, stageId, states, envelopes) {
|
|
112
|
+
const node = dag.nodes.find((n) => n.id === stageId);
|
|
113
|
+
if (!node)
|
|
114
|
+
return false;
|
|
115
|
+
const edges = predecessorEdges(node);
|
|
116
|
+
if (edges.length === 0)
|
|
117
|
+
return true;
|
|
118
|
+
if (expandJoinParents(dag, edges).length >= 2) {
|
|
119
|
+
return multiParentJoinAllowsRun(dag, node, states, envelopes);
|
|
120
|
+
}
|
|
121
|
+
const edge = edges[0];
|
|
122
|
+
const parentNeedId = typeof node.needs === "string" && node.needs ? node.needs : edge.id;
|
|
123
|
+
const parentId = expandJoinParent(dag, parentNeedId)[0];
|
|
124
|
+
if (states.get(parentId) !== "succeeded")
|
|
125
|
+
return false;
|
|
126
|
+
if (edge.if === undefined)
|
|
127
|
+
return true;
|
|
128
|
+
return inboundEdgeFired(edge, parentId, states, envelopes);
|
|
129
|
+
}
|
|
130
|
+
export function classifyInboundAfterSuccess(node, parentId, payload) {
|
|
131
|
+
const inbound = predecessorEdges(node).find((edge) => edge.id === parentId);
|
|
132
|
+
if (inbound?.if === undefined)
|
|
133
|
+
return "ungated";
|
|
134
|
+
return evaluateRouteIf(inbound.if, payload);
|
|
135
|
+
}
|
|
136
|
+
export function isEagerSingleParentIfSkip(node, parentId, payload) {
|
|
137
|
+
if (predecessorEdges(node).length !== 1)
|
|
138
|
+
return false;
|
|
139
|
+
return classifyInboundAfterSuccess(node, parentId, payload) === "miss";
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Finds pending multi-parent joins that cannot run once every parent is
|
|
143
|
+
* terminal: all-skipped, or all-succeeded with an inbound `if` miss.
|
|
144
|
+
* A failed parent is not a stalled skip — that join stays pending.
|
|
145
|
+
*/
|
|
146
|
+
export function pickStalledJoinSkips(dag, states, completedEnvelopes = new Map()) {
|
|
147
|
+
const ids = [];
|
|
148
|
+
for (const node of dag.nodes) {
|
|
149
|
+
if (states.get(node.id) !== "pending")
|
|
150
|
+
continue;
|
|
151
|
+
if (isStalledMultiParentJoin(dag, node, states, completedEnvelopes)) {
|
|
152
|
+
ids.push(node.id);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return ids;
|
|
156
|
+
}
|
|
@@ -3,7 +3,6 @@ import type { RunPipelineDagSnapshot, RunStore, StageSnapshot } from "../runstor
|
|
|
3
3
|
import type { StageEnvelope } from "../types/envelope.js";
|
|
4
4
|
import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
|
|
5
5
|
import type { TaskFile } from "../types/task.js";
|
|
6
|
-
import { applyCloneForksFromEnvelopes, applyCloneForksToSchedule } from "./cloneSchedule.js";
|
|
7
6
|
import type { FeedbackLoopDecisionInput, ResolveFeedbackLoopDecisionResult } from "./feedbackLoopDecision.js";
|
|
8
7
|
import type { PipelineRunResult } from "./pipelineRunner.js";
|
|
9
8
|
import type { StageProcessLauncher } from "./stageProcessLauncher.js";
|
|
@@ -49,8 +48,6 @@ export type HydratedSchedule = {
|
|
|
49
48
|
firstFailureReason: string | undefined;
|
|
50
49
|
dag?: RunPipelineDagSnapshot;
|
|
51
50
|
};
|
|
52
|
-
export { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, };
|
|
53
|
-
export type { ApplyCloneForksResult } from "./cloneSchedule.js";
|
|
54
51
|
export declare function snapshotToScheduleState(snap: {
|
|
55
52
|
status: StageSnapshot["status"];
|
|
56
53
|
}, overrides: ReadonlyMap<string, StageScheduleState>, stageId: string, executionMode?: StageExecutionMode): StageScheduleState;
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { normalizeForkChoice } from "../envelope/forkChoice.js";
|
|
2
2
|
import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
|
|
3
3
|
import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
|
|
4
|
-
import {
|
|
4
|
+
import { cloneFailureContinuesSchedule, cloneRetryDownstream, cloneFailFastSkipIds, cloneScheduleAllowsRun, failureIsAccepted, isCloneInstance, mint, protectedClonableChildIds, sequentialLaterCloneIds, skipRejectedNeedDependents, } from "./cloneSchedule.js";
|
|
5
5
|
import { buildCompletedEnvelopesFromRun, buildStageConfigById, } from "./envelopeRouting.js";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { classifyInboundAfterSuccess, isEagerSingleParentIfSkip, joinAllowsRun, pickStalledJoinSkips, } from "./joinReadiness.js";
|
|
7
|
+
import { forkChoicePreservesReplaySource, mintCohortForFanout, } from "./forkGeneration.js";
|
|
8
|
+
import { activeReplayId, activeSourceStageId, applySendBack, createReplaySchedule, hydrateFromStore, isHeld, launchAttemptFor, launchFor, noteActiveCohort, onContinued, onRouteStageFailed, onRouteStageRunning, onRouteStageSucceeded, rebindRouteAttempts, resolveDecision, } from "./replayLifecycle.js";
|
|
8
9
|
import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
|
|
9
10
|
import { runStage, isRunStageWaiting } from "./stageRunner.js";
|
|
10
11
|
import { attemptContext, resumeSessionFilePath } from "./stageAttemptContext.js";
|
|
11
|
-
const EMPTY_RETRY_DOWNSTREAM = new Set();
|
|
12
12
|
function asDagSnapshot(dag) {
|
|
13
13
|
const snapshot = dag;
|
|
14
14
|
if (Array.isArray(snapshot.stage_ids))
|
|
@@ -18,7 +18,6 @@ function asDagSnapshot(dag) {
|
|
|
18
18
|
stage_ids: dag.nodes.map((n) => n.id),
|
|
19
19
|
};
|
|
20
20
|
}
|
|
21
|
-
export { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, };
|
|
22
21
|
function synthesizedFailureEnvelope(reason) {
|
|
23
22
|
return { status: "failure", summary: reason, artifacts: [] };
|
|
24
23
|
}
|
|
@@ -284,21 +283,9 @@ function isRunnable(dag, stageId, states, schedulingHalted, completedEnvelopes,
|
|
|
284
283
|
const state = states.get(stageId);
|
|
285
284
|
if (state !== "pending")
|
|
286
285
|
return false;
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
if (override === undefined || override.size === 0) {
|
|
291
|
-
return { kind: "untracked" };
|
|
292
|
-
}
|
|
293
|
-
const forkParentId = forkParentForNeedsStage(dag, needsId);
|
|
294
|
-
if (forkParentId === null)
|
|
295
|
-
return { kind: "untracked" };
|
|
296
|
-
const tracked = override.get(forkParentId);
|
|
297
|
-
if (tracked === undefined)
|
|
298
|
-
return { kind: "untracked" };
|
|
299
|
-
return activeCohortFromCloneIds(tracked);
|
|
300
|
-
},
|
|
301
|
-
});
|
|
286
|
+
if (!joinAllowsRun(dag, stageId, states, completedEnvelopes))
|
|
287
|
+
return false;
|
|
288
|
+
return cloneScheduleAllowsRun(dag, stageId, states, completedEnvelopes);
|
|
302
289
|
}
|
|
303
290
|
export function hydratedScheduleHasRunnableWork(hydrated) {
|
|
304
291
|
if (hydrated.schedulingHalted)
|
|
@@ -403,13 +390,6 @@ export async function runPipelineDag(options) {
|
|
|
403
390
|
}
|
|
404
391
|
}
|
|
405
392
|
applyForkSkipsFromEnvelopes(dag, states, completedEnvelopes);
|
|
406
|
-
{
|
|
407
|
-
const applied = applyCloneForksFromEnvelopes(dag, states, completedEnvelopes);
|
|
408
|
-
if (applied.dag !== dag) {
|
|
409
|
-
await store.updatePipelineDag(run.runId, applied.dag);
|
|
410
|
-
}
|
|
411
|
-
dag = applied.dag;
|
|
412
|
-
}
|
|
413
393
|
let activeCount = 0;
|
|
414
394
|
const inFlight = new Set();
|
|
415
395
|
const launchedIds = new Set();
|
|
@@ -442,6 +422,28 @@ export async function runPipelineDag(options) {
|
|
|
442
422
|
await store.appendStageEvent(run.runId, id, { event: "skipped" });
|
|
443
423
|
await persistSkipRejected(id, "skipped");
|
|
444
424
|
};
|
|
425
|
+
/**
|
|
426
|
+
* Finalizes multi-parent (implicit-AND) join nodes that are permanently
|
|
427
|
+
* stuck in `pending`: every predecessor edge has reached a terminal state,
|
|
428
|
+
* but `joinAllowsRun` still can't be satisfied, and terminal
|
|
429
|
+
* states never change back. `shouldSkipForObservedNeed` deliberately
|
|
430
|
+
* refuses to decide these nodes from a single resolving parent (see its
|
|
431
|
+
* comment in cloneSchedule.ts), so this is the only place their fate gets
|
|
432
|
+
* sealed -- once we're sure every sibling has actually settled. Runs once
|
|
433
|
+
* per scheduler tick; loops until a pass finds nothing new, so a chain of
|
|
434
|
+
* stalled joins settles within a single tick rather than one per ~25ms
|
|
435
|
+
* poll.
|
|
436
|
+
*/
|
|
437
|
+
const applyStalledJoinSkips = async () => {
|
|
438
|
+
for (;;) {
|
|
439
|
+
const ids = pickStalledJoinSkips(dag, states, completedEnvelopes);
|
|
440
|
+
if (ids.length === 0)
|
|
441
|
+
return;
|
|
442
|
+
for (const id of ids) {
|
|
443
|
+
await persistSkipPending(id);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
};
|
|
445
447
|
for (const [stageId, state] of states) {
|
|
446
448
|
if (state !== "failed")
|
|
447
449
|
continue;
|
|
@@ -569,22 +571,33 @@ export async function runPipelineDag(options) {
|
|
|
569
571
|
}
|
|
570
572
|
};
|
|
571
573
|
const onStageSuccess = async (stageId, envelope) => {
|
|
572
|
-
const retryDownstreamIds = retryContext
|
|
573
|
-
? cloneRetryDownstream(dag, [...retryContext.retryRoots.keys()])
|
|
574
|
-
: EMPTY_RETRY_DOWNSTREAM;
|
|
575
|
-
const forceFreshCloneIds = activeReplayId(feedbackSchedule) !== undefined;
|
|
576
|
-
const conflict = cloneFanoutConflict(dag, envelope, states, retryDownstreamIds, { forceFreshCloneIds });
|
|
577
|
-
if (conflict !== undefined) {
|
|
578
|
-
await store.appendStageEvent(run.runId, stageId, {
|
|
579
|
-
event: "failed",
|
|
580
|
-
reason: conflict,
|
|
581
|
-
});
|
|
582
|
-
await onStageFailure(stageId, conflict);
|
|
583
|
-
return;
|
|
584
|
-
}
|
|
585
574
|
states.set(stageId, "succeeded");
|
|
586
575
|
completedEnvelopes.set(stageId, envelope);
|
|
587
576
|
notifyRetryRootTerminal(stageId, "succeeded");
|
|
577
|
+
const minted = mint(dag, stageId, envelope);
|
|
578
|
+
if (minted.kind === "error") {
|
|
579
|
+
await handlePostSuccessError(stageId, minted.reason);
|
|
580
|
+
return;
|
|
581
|
+
}
|
|
582
|
+
if (minted.kind === "minted") {
|
|
583
|
+
dag = minted.dag;
|
|
584
|
+
await store.updatePipelineDag(run.runId, dag);
|
|
585
|
+
states.delete(minted.catalogChildId);
|
|
586
|
+
for (const id of minted.instanceIds) {
|
|
587
|
+
states.set(id, "pending");
|
|
588
|
+
}
|
|
589
|
+
const replayId = activeReplayId(feedbackSchedule);
|
|
590
|
+
if (replayId !== undefined) {
|
|
591
|
+
noteActiveCohort(feedbackSchedule, stageId, minted.instanceIds);
|
|
592
|
+
await mintCohortForFanout({
|
|
593
|
+
store,
|
|
594
|
+
runId: run.runId,
|
|
595
|
+
replayId,
|
|
596
|
+
forkParent: stageId,
|
|
597
|
+
cloneStageIds: minted.instanceIds,
|
|
598
|
+
});
|
|
599
|
+
}
|
|
600
|
+
}
|
|
588
601
|
const feedbackAction = envelope.feedback_loop;
|
|
589
602
|
if (feedbackAction?.action === "send_back") {
|
|
590
603
|
const node = dag.nodes.find((n) => n.id === stageId);
|
|
@@ -699,6 +712,18 @@ export async function runPipelineDag(options) {
|
|
|
699
712
|
return;
|
|
700
713
|
}
|
|
701
714
|
const skippedIds = [];
|
|
715
|
+
// applyForkChoiceToSchedule (and everything it calls, including
|
|
716
|
+
// skipRejectedNeedDependents in cloneSchedule.ts) is fully synchronous
|
|
717
|
+
// -- no `await` anywhere in that call chain. So by the time this call
|
|
718
|
+
// returns, `states` already reflects every one of this stage's
|
|
719
|
+
// unchosen children as terminal, atomically, before the persistence
|
|
720
|
+
// loop below runs its first `await`. A downstream multi-parent join
|
|
721
|
+
// (see pickStalledJoinSkips) that depends on several of these children
|
|
722
|
+
// therefore always decides off fully-consistent in-memory state, even
|
|
723
|
+
// though its own `stage_events` row can end up persisted *between*
|
|
724
|
+
// two of these sequential appendStageEvent calls below (this loop) --
|
|
725
|
+
// that interleaving is only a SQLite write-ordering artifact, not a
|
|
726
|
+
// sign the join decided early.
|
|
702
727
|
applyForkChoiceToSchedule(dag, stageId, chosen, states, {
|
|
703
728
|
protectedIds,
|
|
704
729
|
onSkip: (id) => {
|
|
@@ -710,31 +735,21 @@ export async function runPipelineDag(options) {
|
|
|
710
735
|
await store.appendStageEvent(run.runId, id, { event: "skipped" });
|
|
711
736
|
}
|
|
712
737
|
}
|
|
713
|
-
const
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
continue;
|
|
729
|
-
const replayId = activeReplayId(feedbackSchedule);
|
|
730
|
-
await mintCohortForFanout({
|
|
731
|
-
store,
|
|
732
|
-
runId: run.runId,
|
|
733
|
-
...(replayId !== undefined ? { replayId } : {}),
|
|
734
|
-
forkParent: stageId,
|
|
735
|
-
cloneStageIds,
|
|
736
|
-
});
|
|
737
|
-
noteActiveCohort(feedbackSchedule, stageId, cloneStageIds);
|
|
738
|
+
const childIds = dag.childrenOf[stageId] ?? [];
|
|
739
|
+
for (const childId of childIds) {
|
|
740
|
+
const child = dag.nodes.find((node) => node.id === childId);
|
|
741
|
+
if (!child)
|
|
742
|
+
continue;
|
|
743
|
+
const result = classifyInboundAfterSuccess(child, stageId, envelope.payload);
|
|
744
|
+
if (result === "missing_field") {
|
|
745
|
+
schedulingHalted = true;
|
|
746
|
+
if (firstFailureReason === undefined) {
|
|
747
|
+
firstFailureReason = `stage "${stageId}": route if field missing from payload`;
|
|
748
|
+
}
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
if (isEagerSingleParentIfSkip(child, stageId, envelope.payload)) {
|
|
752
|
+
await persistSkipPending(childId);
|
|
738
753
|
}
|
|
739
754
|
}
|
|
740
755
|
};
|
|
@@ -891,6 +906,7 @@ export async function runPipelineDag(options) {
|
|
|
891
906
|
};
|
|
892
907
|
while (!allTerminal()) {
|
|
893
908
|
drainRetryMutations();
|
|
909
|
+
await applyStalledJoinSkips();
|
|
894
910
|
if (options.onLoopTick !== undefined) {
|
|
895
911
|
await options.onLoopTick();
|
|
896
912
|
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
const NUMERIC_OPS = new Set(["gt", "gte", "lt", "lte"]);
|
|
2
|
+
function isPlainObject(value) {
|
|
3
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4
|
+
}
|
|
5
|
+
function hasOwn(object, key) {
|
|
6
|
+
return Object.prototype.hasOwnProperty.call(object, key);
|
|
7
|
+
}
|
|
8
|
+
function isRouteIfLeaf(predicate) {
|
|
9
|
+
return "op" in predicate && predicate.op !== undefined && typeof predicate.field === "string";
|
|
10
|
+
}
|
|
11
|
+
function readPayloadPath(payload, field) {
|
|
12
|
+
if (payload === undefined)
|
|
13
|
+
return { ok: false };
|
|
14
|
+
const segments = field.split(".");
|
|
15
|
+
let current = payload;
|
|
16
|
+
for (let i = 0; i < segments.length; i++) {
|
|
17
|
+
const segment = segments[i];
|
|
18
|
+
if (!isPlainObject(current) || !hasOwn(current, segment)) {
|
|
19
|
+
return { ok: false };
|
|
20
|
+
}
|
|
21
|
+
current = current[segment];
|
|
22
|
+
if (i < segments.length - 1 && !isPlainObject(current)) {
|
|
23
|
+
return { ok: false };
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return { ok: true, value: current };
|
|
27
|
+
}
|
|
28
|
+
function compareLeaf(op, actual, expected) {
|
|
29
|
+
if (op === "eq")
|
|
30
|
+
return actual === expected ? "fire" : "miss";
|
|
31
|
+
if (op === "ne")
|
|
32
|
+
return actual !== expected ? "fire" : "miss";
|
|
33
|
+
if (NUMERIC_OPS.has(op)) {
|
|
34
|
+
if (typeof actual !== "number" || !Number.isFinite(actual))
|
|
35
|
+
return "miss";
|
|
36
|
+
if (typeof expected !== "number" || !Number.isFinite(expected))
|
|
37
|
+
return "miss";
|
|
38
|
+
if (op === "gt")
|
|
39
|
+
return actual > expected ? "fire" : "miss";
|
|
40
|
+
if (op === "gte")
|
|
41
|
+
return actual >= expected ? "fire" : "miss";
|
|
42
|
+
if (op === "lt")
|
|
43
|
+
return actual < expected ? "fire" : "miss";
|
|
44
|
+
if (op === "lte")
|
|
45
|
+
return actual <= expected ? "fire" : "miss";
|
|
46
|
+
}
|
|
47
|
+
if (op === "in") {
|
|
48
|
+
if (!Array.isArray(expected))
|
|
49
|
+
return "miss";
|
|
50
|
+
return expected.includes(actual) ? "fire" : "miss";
|
|
51
|
+
}
|
|
52
|
+
if (op === "not_in") {
|
|
53
|
+
if (!Array.isArray(expected))
|
|
54
|
+
return "miss";
|
|
55
|
+
return expected.includes(actual) ? "miss" : "fire";
|
|
56
|
+
}
|
|
57
|
+
return "miss";
|
|
58
|
+
}
|
|
59
|
+
export function evaluateRouteIf(predicate, payload) {
|
|
60
|
+
if ("all" in predicate && predicate.all !== undefined) {
|
|
61
|
+
let anyMiss = false;
|
|
62
|
+
for (const child of predicate.all) {
|
|
63
|
+
const result = evaluateRouteIf(child, payload);
|
|
64
|
+
if (result === "missing_field")
|
|
65
|
+
return "missing_field";
|
|
66
|
+
if (result === "miss")
|
|
67
|
+
anyMiss = true;
|
|
68
|
+
}
|
|
69
|
+
return anyMiss ? "miss" : "fire";
|
|
70
|
+
}
|
|
71
|
+
if ("any" in predicate && predicate.any !== undefined) {
|
|
72
|
+
let anyFire = false;
|
|
73
|
+
for (const child of predicate.any) {
|
|
74
|
+
const result = evaluateRouteIf(child, payload);
|
|
75
|
+
if (result === "missing_field")
|
|
76
|
+
return "missing_field";
|
|
77
|
+
if (result === "fire")
|
|
78
|
+
anyFire = true;
|
|
79
|
+
}
|
|
80
|
+
return anyFire ? "fire" : "miss";
|
|
81
|
+
}
|
|
82
|
+
if ("not" in predicate && predicate.not !== undefined) {
|
|
83
|
+
const result = evaluateRouteIf(predicate.not, payload);
|
|
84
|
+
if (result === "missing_field")
|
|
85
|
+
return "missing_field";
|
|
86
|
+
return result === "fire" ? "miss" : "fire";
|
|
87
|
+
}
|
|
88
|
+
if (!isRouteIfLeaf(predicate))
|
|
89
|
+
return "missing_field";
|
|
90
|
+
const read = readPayloadPath(payload, predicate.field);
|
|
91
|
+
if (!read.ok)
|
|
92
|
+
return "missing_field";
|
|
93
|
+
return compareLeaf(predicate.op, read.value, predicate.value);
|
|
94
|
+
}
|
|
@@ -25,10 +25,6 @@ export function succeededStageRetryBlocker(stageSnap) {
|
|
|
25
25
|
stageSnap.definition_id !== stageSnap.stage_id) {
|
|
26
26
|
return `Cannot re-run a succeeded clone instance stage (stage=${stageSnap.stage_id}, definition=${stageSnap.definition_id}): clone instances are created by upstream fan-out and cannot be rebuilt in place. Use rerun to start a new run.`;
|
|
27
27
|
}
|
|
28
|
-
const forks = stageSnap.envelope?.clone_forks;
|
|
29
|
-
if (Array.isArray(forks) && forks.length > 0) {
|
|
30
|
-
return `Cannot re-run a succeeded clonable-fanout stage (stage=${stageSnap.stage_id}): its envelope declared clone_forks and existing clone instances cannot be pruned. Use rerun to start a new run.`;
|
|
31
|
-
}
|
|
32
28
|
if (stageSnap.envelope?.fork_choice !== undefined) {
|
|
33
29
|
return `Cannot re-run a succeeded fork stage (stage=${stageSnap.stage_id}): its envelope recorded a fork_choice and skipped branches cannot be restored. Use rerun to start a new run.`;
|
|
34
30
|
}
|