stageflow 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +190 -13
- package/dist/agent/activity.d.ts +10 -1
- package/dist/agent/activity.js +37 -2
- package/dist/agent/activityObserver.d.ts +5 -1
- package/dist/agent/activityObserver.js +174 -11
- package/dist/agent/fakeAgent.js +4 -0
- package/dist/agent/piAdapter.d.ts +16 -0
- package/dist/agent/piAdapter.js +88 -21
- package/dist/agent/port.d.ts +2 -0
- package/dist/agent/providerAuth.js +12 -7
- package/dist/catalog/displayCatalogPath.d.ts +12 -0
- package/dist/catalog/displayCatalogPath.js +28 -0
- package/dist/cli/ciIdentity.d.ts +14 -0
- package/dist/cli/ciIdentity.js +52 -0
- package/dist/cli/initCommand.d.ts +9 -0
- package/dist/cli/initCommand.js +71 -0
- package/dist/cli/initTemplates.d.ts +3 -0
- package/dist/cli/initTemplates.js +19 -0
- package/dist/cli/operatorCatalog.d.ts +10 -0
- package/dist/cli/operatorCatalog.js +16 -0
- package/dist/cli/runCommand.d.ts +26 -0
- package/dist/cli/runCommand.js +209 -0
- package/dist/cli/runOutput.d.ts +19 -0
- package/dist/cli/runOutput.js +125 -0
- package/dist/cli/validateCommand.d.ts +1 -1
- package/dist/cli/validateCommand.js +20 -3
- package/dist/cli.d.ts +12 -1
- package/dist/cli.js +82 -66
- package/dist/config/browseCatalog.d.ts +45 -0
- package/dist/config/browseCatalog.js +170 -0
- package/dist/config/createPipeline.d.ts +12 -3
- package/dist/config/createPipeline.js +292 -100
- package/dist/config/createStage.d.ts +10 -4
- package/dist/config/createStage.js +44 -14
- package/dist/config/listConfig.d.ts +14 -19
- package/dist/config/listConfig.js +26 -191
- package/dist/config/listModelsFromManifest.d.ts +2 -0
- package/dist/config/listModelsFromManifest.js +4 -0
- package/dist/config/loadOutcome.d.ts +4 -0
- package/dist/config/loadPipeline.d.ts +1 -15
- package/dist/config/loadPipeline.js +84 -142
- package/dist/config/loadStage.d.ts +4 -0
- package/dist/config/loadStage.js +60 -32
- package/dist/config/loadStageflowManifest.d.ts +5 -0
- package/dist/config/loadStageflowManifest.js +157 -0
- package/dist/config/mergePipelineIncludes.d.ts +9 -0
- package/dist/config/mergePipelineIncludes.js +141 -0
- package/dist/config/normalizePipelineStageEntry.d.ts +16 -0
- package/dist/config/normalizePipelineStageEntry.js +186 -0
- package/dist/config/pipelineStageKeys.d.ts +4 -0
- package/dist/config/pipelineStageKeys.js +14 -0
- package/dist/config/resolveCatalogContext.d.ts +13 -0
- package/dist/config/resolveCatalogContext.js +13 -0
- package/dist/config/resolveForkEmitContext.d.ts +3 -0
- package/dist/config/resolveForkEmitContext.js +12 -0
- package/dist/config/resolvePipelineDag.d.ts +5 -1
- package/dist/config/resolvePipelineDag.js +54 -36
- package/dist/config/scanCatalogPaths.d.ts +4 -0
- package/dist/config/scanCatalogPaths.js +83 -0
- package/dist/config/validateCatalog.d.ts +22 -4
- package/dist/config/validateCatalog.js +169 -132
- package/dist/envelope/check.js +7 -0
- package/dist/envelope/forkChoice.d.ts +3 -0
- package/dist/envelope/forkChoice.js +47 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/mcp/projectRun.d.ts +3 -0
- package/dist/mcp/projectRun.js +7 -0
- package/dist/mcp/server.js +1 -1
- package/dist/mcp/tools.js +28 -11
- package/dist/project/findProjectRoot.d.ts +2 -0
- package/dist/project/findProjectRoot.js +51 -0
- package/dist/project/globalHome.d.ts +2 -0
- package/dist/project/globalHome.js +19 -0
- package/dist/project/resolveProjectContext.d.ts +9 -0
- package/dist/project/resolveProjectContext.js +38 -0
- package/dist/project/resolveStageflowContext.d.ts +11 -0
- package/dist/project/resolveStageflowContext.js +66 -0
- package/dist/runstore/normalizeCatalogPath.d.ts +1 -0
- package/dist/runstore/normalizeCatalogPath.js +4 -0
- package/dist/runstore/port.d.ts +15 -0
- package/dist/runstore/runProjection.js +11 -1
- package/dist/runstore/sqlite/SqliteRunStore.js +46 -4
- package/dist/runstore/sqlite/schema.d.ts +1 -1
- package/dist/runstore/sqlite/schema.js +7 -1
- package/dist/runtime/credentialBinding.d.ts +8 -6
- package/dist/runtime/credentialBinding.js +18 -24
- package/dist/runtime/pipelineRunner.d.ts +11 -0
- package/dist/runtime/pipelineRunner.js +25 -1
- package/dist/runtime/pipelineScheduler.d.ts +3 -0
- package/dist/runtime/pipelineScheduler.js +70 -14
- package/dist/runtime/reloadRunCatalog.d.ts +5 -0
- package/dist/runtime/reloadRunCatalog.js +44 -0
- package/dist/runtime/resumeReconstruct.d.ts +1 -0
- package/dist/runtime/resumeReconstruct.js +27 -10
- package/dist/runtime/runManager.d.ts +8 -0
- package/dist/runtime/runManager.js +49 -18
- package/dist/runtime/settingsFile.d.ts +12 -0
- package/dist/runtime/settingsFile.js +87 -6
- package/dist/runtime/stageAttemptBootstrap.js +3 -0
- package/dist/runtime/stageProcessLauncher.d.ts +1 -0
- package/dist/runtime/stageProcessLauncher.js +6 -0
- package/dist/runtime/stageRecovery.js +3 -1
- package/dist/runtime/stageRunner.d.ts +2 -0
- package/dist/runtime/stageRunner.js +9 -2
- package/dist/runtime/stageWorker.js +4 -6
- package/dist/runtime/stageWorkerProtocol.d.ts +1 -0
- package/dist/server/http.js +33 -9
- package/dist/tools/emitStageEnvelope.d.ts +3 -1
- package/dist/tools/emitStageEnvelope.js +11 -1
- package/dist/types/envelope.d.ts +1 -0
- package/dist/types/forkChoice.d.ts +8 -0
- package/dist/types/forkChoice.js +1 -0
- package/dist/types/pipeline.d.ts +51 -2
- package/dist/types/stageflowManifest.d.ts +23 -0
- package/dist/types/stageflowManifest.js +1 -0
- package/dist/ui/assets/index-CFSzDZje.js +118 -0
- package/dist/ui/assets/{index-DCsDopak.css → index-DefBlEvN.css} +1 -1
- package/dist/ui/index.html +3 -2
- package/dist/ui/stageflow-icon.svg +12 -0
- package/package.json +7 -5
- package/dist/agent/cursorExtension.d.ts +0 -12
- package/dist/agent/cursorExtension.js +0 -88
- package/dist/runstore/catalog.d.ts +0 -8
- package/dist/runstore/catalog.js +0 -16
- package/dist/runstore/disk/DiskRunStore.d.ts +0 -30
- package/dist/runstore/disk/DiskRunStore.js +0 -238
- package/dist/runstore/layout.d.ts +0 -22
- package/dist/runstore/layout.js +0 -58
- package/dist/runtime/hitlSeams.d.ts +0 -38
- package/dist/runtime/hitlSeams.js +0 -2
- package/dist/ui/assets/index-Cry0Tpfx.js +0 -118
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { loadFailure, loadSuccess } from "./loadOutcome.js";
|
|
3
|
+
import { readYamlObject } from "./readYamlObject.js";
|
|
4
|
+
function isPlainObject(value) {
|
|
5
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
function normalizePath(filePath) {
|
|
8
|
+
return path.normalize(path.resolve(filePath));
|
|
9
|
+
}
|
|
10
|
+
function formatIncludeCycle(stack, nextPath) {
|
|
11
|
+
const chain = [...stack, nextPath].map((p) => path.basename(p));
|
|
12
|
+
return chain.join(" → ");
|
|
13
|
+
}
|
|
14
|
+
async function visitPipelineFile(filePath, stack, idLocations, entries) {
|
|
15
|
+
const absPath = normalizePath(filePath);
|
|
16
|
+
if (stack.some((p) => normalizePath(p) === absPath)) {
|
|
17
|
+
const cycleChain = formatIncludeCycle(stack, absPath);
|
|
18
|
+
return loadFailure([
|
|
19
|
+
{
|
|
20
|
+
code: "pipeline.include_cycle",
|
|
21
|
+
message: `Include cycle detected: ${cycleChain}`,
|
|
22
|
+
category: "pipeline",
|
|
23
|
+
},
|
|
24
|
+
]);
|
|
25
|
+
}
|
|
26
|
+
let raw;
|
|
27
|
+
try {
|
|
28
|
+
raw = await readYamlObject(absPath);
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
32
|
+
return loadFailure([
|
|
33
|
+
{
|
|
34
|
+
code: "pipeline.load_error",
|
|
35
|
+
message,
|
|
36
|
+
category: "pipeline",
|
|
37
|
+
},
|
|
38
|
+
]);
|
|
39
|
+
}
|
|
40
|
+
const nextStack = [...stack, absPath];
|
|
41
|
+
if (raw.include !== undefined) {
|
|
42
|
+
if (!Array.isArray(raw.include)) {
|
|
43
|
+
return loadFailure([
|
|
44
|
+
{
|
|
45
|
+
code: "pipeline.include_invalid",
|
|
46
|
+
message: `Invalid include in ${absPath}: include must be an array`,
|
|
47
|
+
category: "pipeline",
|
|
48
|
+
},
|
|
49
|
+
]);
|
|
50
|
+
}
|
|
51
|
+
for (let index = 0; index < raw.include.length; index++) {
|
|
52
|
+
const item = raw.include[index];
|
|
53
|
+
if (!isPlainObject(item) || typeof item.local !== "string" || !item.local) {
|
|
54
|
+
return loadFailure([
|
|
55
|
+
{
|
|
56
|
+
code: "pipeline.include_invalid",
|
|
57
|
+
message: `Invalid include entry at index ${index} in ${absPath}: expected { local: "<path>" }`,
|
|
58
|
+
category: "pipeline",
|
|
59
|
+
},
|
|
60
|
+
]);
|
|
61
|
+
}
|
|
62
|
+
const includePath = normalizePath(path.resolve(path.dirname(absPath), item.local));
|
|
63
|
+
const includeResult = await visitPipelineFile(includePath, nextStack, idLocations, entries);
|
|
64
|
+
if (!includeResult.ok)
|
|
65
|
+
return includeResult;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (Array.isArray(raw.stages)) {
|
|
69
|
+
for (const entry of raw.stages) {
|
|
70
|
+
if (isPlainObject(entry)) {
|
|
71
|
+
const entryId = typeof entry.id === "string" ? entry.id : undefined;
|
|
72
|
+
if (entryId) {
|
|
73
|
+
const priorPath = idLocations.get(entryId);
|
|
74
|
+
if (priorPath) {
|
|
75
|
+
return loadFailure([
|
|
76
|
+
{
|
|
77
|
+
code: "pipeline.include_duplicate_stage",
|
|
78
|
+
message: `Duplicate stage id "${entryId}" in ${absPath} (also declared in ${priorPath})`,
|
|
79
|
+
category: "pipeline",
|
|
80
|
+
},
|
|
81
|
+
]);
|
|
82
|
+
}
|
|
83
|
+
idLocations.set(entryId, absPath);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
entries.push({ raw: entry, declaringPath: absPath });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return loadSuccess(undefined);
|
|
90
|
+
}
|
|
91
|
+
export async function mergePipelineStages(rootPath) {
|
|
92
|
+
const absRoot = normalizePath(rootPath);
|
|
93
|
+
let raw;
|
|
94
|
+
try {
|
|
95
|
+
raw = await readYamlObject(absRoot);
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
99
|
+
return loadFailure([
|
|
100
|
+
{
|
|
101
|
+
code: "pipeline.load_error",
|
|
102
|
+
message,
|
|
103
|
+
category: "pipeline",
|
|
104
|
+
},
|
|
105
|
+
]);
|
|
106
|
+
}
|
|
107
|
+
if (typeof raw.id !== "string" || !raw.id) {
|
|
108
|
+
return loadFailure([
|
|
109
|
+
{
|
|
110
|
+
code: "pipeline.invalid_shape",
|
|
111
|
+
message: `Invalid pipeline ${absRoot}: id is required`,
|
|
112
|
+
category: "pipeline",
|
|
113
|
+
},
|
|
114
|
+
]);
|
|
115
|
+
}
|
|
116
|
+
if (!Array.isArray(raw.stages)) {
|
|
117
|
+
return loadFailure([
|
|
118
|
+
{
|
|
119
|
+
code: "pipeline.invalid_shape",
|
|
120
|
+
message: `Invalid pipeline ${absRoot}: stages[] is required`,
|
|
121
|
+
category: "pipeline",
|
|
122
|
+
},
|
|
123
|
+
]);
|
|
124
|
+
}
|
|
125
|
+
const pipelineId = raw.id;
|
|
126
|
+
const idLocations = new Map();
|
|
127
|
+
const entries = [];
|
|
128
|
+
const mergeResult = await visitPipelineFile(absRoot, [], idLocations, entries);
|
|
129
|
+
if (!mergeResult.ok)
|
|
130
|
+
return mergeResult;
|
|
131
|
+
if (entries.length === 0) {
|
|
132
|
+
return loadFailure([
|
|
133
|
+
{
|
|
134
|
+
code: "pipeline.invalid_shape",
|
|
135
|
+
message: `Invalid pipeline ${absRoot}: stages must be non-empty`,
|
|
136
|
+
category: "pipeline",
|
|
137
|
+
},
|
|
138
|
+
]);
|
|
139
|
+
}
|
|
140
|
+
return loadSuccess({ entries, pipelineId });
|
|
141
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { NormalizedPipelineStageEntry } from "../types/pipeline.js";
|
|
2
|
+
import { type LoadOutcome } from "./loadOutcome.js";
|
|
3
|
+
import type { RawMergedEntry } from "./mergePipelineIncludes.js";
|
|
4
|
+
export declare function inferIdFromUsesPath(usesPath: string): string | null;
|
|
5
|
+
export declare function normalizePipelineStageEntries(rawEntries: RawMergedEntry[], ctx: {
|
|
6
|
+
pipelineId: string;
|
|
7
|
+
path: string;
|
|
8
|
+
}): LoadOutcome<NormalizedPipelineStageEntry[]>;
|
|
9
|
+
export declare function toWiringRefs(entries: NormalizedPipelineStageEntry[]): Array<{
|
|
10
|
+
id: string;
|
|
11
|
+
needs?: string;
|
|
12
|
+
fork?: {
|
|
13
|
+
select: "one" | "subset";
|
|
14
|
+
allow_none?: boolean;
|
|
15
|
+
};
|
|
16
|
+
}>;
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { loadFailure, loadSuccess } from "./loadOutcome.js";
|
|
3
|
+
import { BODY_KEYS, isAllowedPipelineStageEntryKey, } from "./pipelineStageKeys.js";
|
|
4
|
+
function isPlainObject(value) {
|
|
5
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
export function inferIdFromUsesPath(usesPath) {
|
|
8
|
+
const base = path.basename(usesPath);
|
|
9
|
+
if (!base)
|
|
10
|
+
return null;
|
|
11
|
+
if (base.endsWith(".stage.yaml")) {
|
|
12
|
+
const id = base.slice(0, -".stage.yaml".length);
|
|
13
|
+
return id || null;
|
|
14
|
+
}
|
|
15
|
+
if (base.endsWith(".yaml")) {
|
|
16
|
+
const id = base.slice(0, -".yaml".length);
|
|
17
|
+
return id || null;
|
|
18
|
+
}
|
|
19
|
+
return base;
|
|
20
|
+
}
|
|
21
|
+
function extractBodyRaw(raw) {
|
|
22
|
+
const body = {};
|
|
23
|
+
for (const key of Object.keys(raw)) {
|
|
24
|
+
if (BODY_KEYS.has(key)) {
|
|
25
|
+
body[key] = raw[key];
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return body;
|
|
29
|
+
}
|
|
30
|
+
function hasBodyKey(raw) {
|
|
31
|
+
return Object.keys(raw).some((key) => BODY_KEYS.has(key));
|
|
32
|
+
}
|
|
33
|
+
function stringStageRefMessage(index, declaringPath, entry) {
|
|
34
|
+
const hint = entry
|
|
35
|
+
? `bare string stage refs are not supported; use { id: "${entry}", uses: "./${entry}.yaml" } or inline body`
|
|
36
|
+
: `bare string stage refs are not supported; use { id: "…", uses: "./….yaml" } or inline body`;
|
|
37
|
+
return `Invalid stage entry at index ${index} in ${declaringPath}: ${hint}`;
|
|
38
|
+
}
|
|
39
|
+
export function normalizePipelineStageEntries(rawEntries, ctx) {
|
|
40
|
+
const normalized = [];
|
|
41
|
+
for (let index = 0; index < rawEntries.length; index++) {
|
|
42
|
+
const { raw, declaringPath } = rawEntries[index];
|
|
43
|
+
if (typeof raw === "string") {
|
|
44
|
+
return loadFailure([
|
|
45
|
+
{
|
|
46
|
+
code: "pipeline.string_stage_ref",
|
|
47
|
+
message: stringStageRefMessage(index, declaringPath, raw),
|
|
48
|
+
category: "pipeline",
|
|
49
|
+
pipelineId: ctx.pipelineId,
|
|
50
|
+
},
|
|
51
|
+
]);
|
|
52
|
+
}
|
|
53
|
+
if (!isPlainObject(raw)) {
|
|
54
|
+
return loadFailure([
|
|
55
|
+
{
|
|
56
|
+
code: "pipeline.invalid_shape",
|
|
57
|
+
message: `Invalid stage entry at index ${index} in ${declaringPath}: expected object`,
|
|
58
|
+
category: "pipeline",
|
|
59
|
+
pipelineId: ctx.pipelineId,
|
|
60
|
+
},
|
|
61
|
+
]);
|
|
62
|
+
}
|
|
63
|
+
for (const key of Object.keys(raw)) {
|
|
64
|
+
if (!isAllowedPipelineStageEntryKey(key)) {
|
|
65
|
+
return loadFailure([
|
|
66
|
+
{
|
|
67
|
+
code: "pipeline.invalid_shape",
|
|
68
|
+
message: `Pipeline ${ctx.pipelineId} (${ctx.path}): invalid stage entry at index ${index}: unknown key "${key}"`,
|
|
69
|
+
category: "pipeline",
|
|
70
|
+
pipelineId: ctx.pipelineId,
|
|
71
|
+
},
|
|
72
|
+
]);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const uses = typeof raw.uses === "string" ? raw.uses : undefined;
|
|
76
|
+
const hasBody = hasBodyKey(raw);
|
|
77
|
+
if (uses && hasBody) {
|
|
78
|
+
return loadFailure([
|
|
79
|
+
{
|
|
80
|
+
code: "pipeline.stage_uses_inline_conflict",
|
|
81
|
+
message: `Pipeline ${ctx.pipelineId} (${ctx.path}): stage entry at index ${index} has both uses and inline body fields`,
|
|
82
|
+
category: "pipeline",
|
|
83
|
+
pipelineId: ctx.pipelineId,
|
|
84
|
+
},
|
|
85
|
+
]);
|
|
86
|
+
}
|
|
87
|
+
if (!uses && !hasBody) {
|
|
88
|
+
const entryLabel = typeof raw.id === "string" ? raw.id : String(index);
|
|
89
|
+
return loadFailure([
|
|
90
|
+
{
|
|
91
|
+
code: "pipeline.stage_missing_body",
|
|
92
|
+
message: `Pipeline ${ctx.pipelineId} (${ctx.path}): stage "${entryLabel}" has no uses: path or inline body`,
|
|
93
|
+
category: "pipeline",
|
|
94
|
+
pipelineId: ctx.pipelineId,
|
|
95
|
+
},
|
|
96
|
+
]);
|
|
97
|
+
}
|
|
98
|
+
let id = typeof raw.id === "string" && raw.id ? raw.id : undefined;
|
|
99
|
+
if (!id && uses) {
|
|
100
|
+
const inferred = inferIdFromUsesPath(uses);
|
|
101
|
+
if (inferred)
|
|
102
|
+
id = inferred;
|
|
103
|
+
}
|
|
104
|
+
if (!id) {
|
|
105
|
+
return loadFailure([
|
|
106
|
+
{
|
|
107
|
+
code: "pipeline.invalid_shape",
|
|
108
|
+
message: `Pipeline ${ctx.pipelineId} (${ctx.path}): invalid stage entry at index ${index}: id must be a non-empty string or inferrable from uses path`,
|
|
109
|
+
category: "pipeline",
|
|
110
|
+
pipelineId: ctx.pipelineId,
|
|
111
|
+
},
|
|
112
|
+
]);
|
|
113
|
+
}
|
|
114
|
+
if (raw.needs !== undefined) {
|
|
115
|
+
if (Array.isArray(raw.needs)) {
|
|
116
|
+
return loadFailure([
|
|
117
|
+
{
|
|
118
|
+
code: "pipeline.dag_error",
|
|
119
|
+
message: `Pipeline ${ctx.pipelineId} (${ctx.path}): stage "${id}": needs must be a string, not an array`,
|
|
120
|
+
category: "pipeline",
|
|
121
|
+
pipelineId: ctx.pipelineId,
|
|
122
|
+
},
|
|
123
|
+
]);
|
|
124
|
+
}
|
|
125
|
+
if (typeof raw.needs !== "string" || !raw.needs) {
|
|
126
|
+
return loadFailure([
|
|
127
|
+
{
|
|
128
|
+
code: "pipeline.dag_error",
|
|
129
|
+
message: `Pipeline ${ctx.pipelineId} (${ctx.path}): stage "${id}": needs must be a non-empty string`,
|
|
130
|
+
category: "pipeline",
|
|
131
|
+
pipelineId: ctx.pipelineId,
|
|
132
|
+
},
|
|
133
|
+
]);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
let forkValue;
|
|
137
|
+
if (raw.fork !== undefined) {
|
|
138
|
+
if (!isPlainObject(raw.fork)) {
|
|
139
|
+
return loadFailure([
|
|
140
|
+
{
|
|
141
|
+
code: "pipeline.dag_error",
|
|
142
|
+
message: `Pipeline ${ctx.pipelineId} (${ctx.path}): stage "${id}": fork must be an object`,
|
|
143
|
+
category: "pipeline",
|
|
144
|
+
pipelineId: ctx.pipelineId,
|
|
145
|
+
},
|
|
146
|
+
]);
|
|
147
|
+
}
|
|
148
|
+
forkValue = raw.fork;
|
|
149
|
+
}
|
|
150
|
+
let body;
|
|
151
|
+
if (uses) {
|
|
152
|
+
const absolutePath = path.resolve(path.dirname(declaringPath), uses);
|
|
153
|
+
body = { kind: "uses", path: uses, absolutePath };
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
body = { kind: "inline", raw: extractBodyRaw(raw) };
|
|
157
|
+
}
|
|
158
|
+
const entry = {
|
|
159
|
+
id,
|
|
160
|
+
declaringPath,
|
|
161
|
+
body,
|
|
162
|
+
...(typeof raw.needs === "string" ? { needs: raw.needs } : {}),
|
|
163
|
+
...(forkValue !== undefined ? { fork: forkValue } : {}),
|
|
164
|
+
};
|
|
165
|
+
const priorPath = normalized.find((e) => e.id === id)?.declaringPath;
|
|
166
|
+
if (priorPath) {
|
|
167
|
+
return loadFailure([
|
|
168
|
+
{
|
|
169
|
+
code: "pipeline.include_duplicate_stage",
|
|
170
|
+
message: `Duplicate stage id "${id}" in ${declaringPath} (also declared in ${priorPath})`,
|
|
171
|
+
category: "pipeline",
|
|
172
|
+
pipelineId: ctx.pipelineId,
|
|
173
|
+
},
|
|
174
|
+
]);
|
|
175
|
+
}
|
|
176
|
+
normalized.push(entry);
|
|
177
|
+
}
|
|
178
|
+
return loadSuccess(normalized);
|
|
179
|
+
}
|
|
180
|
+
export function toWiringRefs(entries) {
|
|
181
|
+
return entries.map((entry) => ({
|
|
182
|
+
id: entry.id,
|
|
183
|
+
...(entry.needs !== undefined ? { needs: entry.needs } : {}),
|
|
184
|
+
...(entry.fork !== undefined ? { fork: entry.fork } : {}),
|
|
185
|
+
}));
|
|
186
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export const WIRING_KEYS = new Set(["id", "needs", "fork", "uses"]);
|
|
2
|
+
export const BODY_KEYS = new Set([
|
|
3
|
+
"system_prompt",
|
|
4
|
+
"model",
|
|
5
|
+
"payload_schema",
|
|
6
|
+
"gate_kinds",
|
|
7
|
+
"skill",
|
|
8
|
+
]);
|
|
9
|
+
export function isPipelineStageBodyKey(key) {
|
|
10
|
+
return BODY_KEYS.has(key);
|
|
11
|
+
}
|
|
12
|
+
export function isAllowedPipelineStageEntryKey(key) {
|
|
13
|
+
return WIRING_KEYS.has(key) || BODY_KEYS.has(key);
|
|
14
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { LoadedManifest } from "../types/stageflowManifest.js";
|
|
2
|
+
import type { LoadIssue } from "./loadOutcome.js";
|
|
3
|
+
import { type CatalogManifestStatus, type StageflowContext } from "../project/resolveStageflowContext.js";
|
|
4
|
+
export type { CatalogManifestStatus };
|
|
5
|
+
export type CatalogContext = {
|
|
6
|
+
projectRoot: string | null;
|
|
7
|
+
manifest: LoadedManifest | null;
|
|
8
|
+
manifestStatus: CatalogManifestStatus;
|
|
9
|
+
issues: LoadIssue[];
|
|
10
|
+
};
|
|
11
|
+
export declare function catalogContextFromStageflow(ctx: StageflowContext): CatalogContext;
|
|
12
|
+
/** @deprecated Prefer resolveStageflowContext */
|
|
13
|
+
export declare function resolveCatalogContext(invocationCwd: string): Promise<CatalogContext>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { resolveStageflowContext, } from "../project/resolveStageflowContext.js";
|
|
2
|
+
export function catalogContextFromStageflow(ctx) {
|
|
3
|
+
return {
|
|
4
|
+
projectRoot: ctx.isGitProject ? ctx.projectRoot : null,
|
|
5
|
+
manifest: ctx.manifest,
|
|
6
|
+
manifestStatus: ctx.manifestStatus,
|
|
7
|
+
issues: ctx.manifestIssues,
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
/** @deprecated Prefer resolveStageflowContext */
|
|
11
|
+
export async function resolveCatalogContext(invocationCwd) {
|
|
12
|
+
return catalogContextFromStageflow(await resolveStageflowContext(invocationCwd));
|
|
13
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export function resolveForkEmitContext(dag, stageId) {
|
|
2
|
+
const dagNode = dag.nodes.find((n) => n.id === stageId);
|
|
3
|
+
if (!dagNode?.fork)
|
|
4
|
+
return undefined;
|
|
5
|
+
return {
|
|
6
|
+
immediateSuccessorIds: dag.childrenOf[stageId] ?? [],
|
|
7
|
+
forkShape: {
|
|
8
|
+
cardinality: dagNode.fork.select,
|
|
9
|
+
allowNone: dagNode.fork.allow_none,
|
|
10
|
+
},
|
|
11
|
+
};
|
|
12
|
+
}
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import type { PipelineStageYamlEntry, ResolvedPipelineDag } from "../types/pipeline.js";
|
|
1
|
+
import type { PipelineStageRef, PipelineStageYamlEntry, ResolvedPipelineDag } from "../types/pipeline.js";
|
|
2
2
|
export type ResolvePipelineDagContext = {
|
|
3
3
|
pipelineId: string;
|
|
4
4
|
path: string;
|
|
5
5
|
};
|
|
6
6
|
export declare function parsePipelineStageEntries(raw: unknown, ctx: ResolvePipelineDagContext): PipelineStageYamlEntry[];
|
|
7
|
+
export declare function resolvePipelineDagFromRefs(refs: PipelineStageRef[], ctx: ResolvePipelineDagContext): {
|
|
8
|
+
stages: string[];
|
|
9
|
+
dag: ResolvedPipelineDag;
|
|
10
|
+
};
|
|
7
11
|
export declare function resolvePipelineDag(rawStages: unknown, ctx: ResolvePipelineDagContext): {
|
|
8
12
|
stages: string[];
|
|
9
13
|
dag: ResolvedPipelineDag;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import { isAllowedPipelineStageEntryKey } from "./pipelineStageKeys.js";
|
|
2
|
+
const ALLOWED_FORK_KEYS = new Set(["select", "allow_none"]);
|
|
2
3
|
function pipelineLabel(ctx) {
|
|
3
4
|
return `Pipeline ${ctx.pipelineId} (${ctx.path})`;
|
|
4
5
|
}
|
|
@@ -19,26 +20,37 @@ export function parsePipelineStageEntries(raw, ctx) {
|
|
|
19
20
|
for (let index = 0; index < raw.length; index++) {
|
|
20
21
|
const entry = raw[index];
|
|
21
22
|
if (typeof entry === "string") {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
continue;
|
|
23
|
+
const hint = entry
|
|
24
|
+
? `invalid stage entry at index ${index}: bare string stage refs are not supported; use { id: "${entry}", uses: "./${entry}.yaml" } or inline body`
|
|
25
|
+
: `invalid stage entry at index ${index}: bare string stage refs are not supported; use { id: "…", uses: "./….yaml" } or inline body`;
|
|
26
|
+
throw new Error(formatError(ctx, hint));
|
|
27
27
|
}
|
|
28
28
|
if (!isPlainObject(entry)) {
|
|
29
29
|
throw new Error(formatError(ctx, `invalid stage entry at index ${index}`));
|
|
30
30
|
}
|
|
31
31
|
const keys = Object.keys(entry);
|
|
32
32
|
for (const key of keys) {
|
|
33
|
-
if (!
|
|
33
|
+
if (!isAllowedPipelineStageEntryKey(key)) {
|
|
34
34
|
throw new Error(formatError(ctx, `invalid stage entry "${String(entry.id ?? index)}": unknown key "${key}"`));
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
if (typeof entry.id !== "string" || !entry.id) {
|
|
38
38
|
throw new Error(formatError(ctx, `invalid stage entry at index ${index}: id must be a non-empty string`));
|
|
39
39
|
}
|
|
40
|
+
let forkValue;
|
|
41
|
+
if (entry.fork !== undefined) {
|
|
42
|
+
if (!isPlainObject(entry.fork)) {
|
|
43
|
+
throw new Error(formatError(ctx, `stage "${entry.id}": fork must be an object`));
|
|
44
|
+
}
|
|
45
|
+
for (const fk of Object.keys(entry.fork)) {
|
|
46
|
+
if (!ALLOWED_FORK_KEYS.has(fk)) {
|
|
47
|
+
throw new Error(formatError(ctx, `stage "${entry.id}": fork: unknown key "${fk}"`));
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
forkValue = entry.fork;
|
|
51
|
+
}
|
|
40
52
|
if (entry.needs === undefined) {
|
|
41
|
-
entries.push({ id: entry.id });
|
|
53
|
+
entries.push(forkValue !== undefined ? { id: entry.id, fork: forkValue } : { id: entry.id });
|
|
42
54
|
continue;
|
|
43
55
|
}
|
|
44
56
|
if (Array.isArray(entry.needs)) {
|
|
@@ -50,29 +62,19 @@ export function parsePipelineStageEntries(raw, ctx) {
|
|
|
50
62
|
if (typeof entry.needs !== "string" || !entry.needs) {
|
|
51
63
|
throw new Error(formatError(ctx, `stage "${entry.id}": needs must be a non-empty string`));
|
|
52
64
|
}
|
|
53
|
-
entries.push(
|
|
65
|
+
entries.push(forkValue !== undefined
|
|
66
|
+
? { id: entry.id, needs: entry.needs, fork: forkValue }
|
|
67
|
+
: { id: entry.id, needs: entry.needs });
|
|
54
68
|
}
|
|
55
69
|
return entries;
|
|
56
70
|
}
|
|
57
71
|
function normalizeToEdges(entries) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
needs: index > 0 ? entries[index - 1] : null,
|
|
65
|
-
stageIndex: index,
|
|
66
|
-
});
|
|
67
|
-
continue;
|
|
68
|
-
}
|
|
69
|
-
edges.push({
|
|
70
|
-
id: entry.id,
|
|
71
|
-
needs: entry.needs ?? null,
|
|
72
|
-
stageIndex: index,
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
return edges;
|
|
72
|
+
return entries.map((entry, index) => ({
|
|
73
|
+
id: entry.id,
|
|
74
|
+
needs: entry.needs ?? null,
|
|
75
|
+
stageIndex: index,
|
|
76
|
+
...(entry.fork !== undefined ? { fork: entry.fork } : {}),
|
|
77
|
+
}));
|
|
76
78
|
}
|
|
77
79
|
function detectDuplicateIds(edges, ctx) {
|
|
78
80
|
const seen = new Set();
|
|
@@ -160,7 +162,7 @@ function topologicalSort(edges) {
|
|
|
160
162
|
indegree.set(edge.id, (indegree.get(edge.id) ?? 0) + 1);
|
|
161
163
|
children.get(edge.needs)?.push(edge.id);
|
|
162
164
|
}
|
|
163
|
-
for (const [
|
|
165
|
+
for (const [, childIds] of children) {
|
|
164
166
|
childIds.sort((a, b) => (byId.get(a)?.stageIndex ?? 0) - (byId.get(b)?.stageIndex ?? 0));
|
|
165
167
|
}
|
|
166
168
|
const roots = edges
|
|
@@ -208,12 +210,26 @@ function buildResolvedPipelineDag(edges) {
|
|
|
208
210
|
needs: edge.needs,
|
|
209
211
|
ancestors: ancestorsById.get(edge.id) ?? [],
|
|
210
212
|
stageIndex: edge.stageIndex,
|
|
213
|
+
...(edge.fork !== undefined
|
|
214
|
+
? { fork: { select: edge.fork.select, allow_none: edge.fork.allow_none ?? false } }
|
|
215
|
+
: {}),
|
|
211
216
|
}));
|
|
212
217
|
return { nodes, roots, childrenOf };
|
|
213
218
|
}
|
|
214
|
-
|
|
215
|
-
const
|
|
216
|
-
|
|
219
|
+
function validateForkFields(edges, dag, ctx) {
|
|
220
|
+
for (const edge of edges) {
|
|
221
|
+
if (!edge.fork)
|
|
222
|
+
continue;
|
|
223
|
+
if (edge.fork.select !== "one" && edge.fork.select !== "subset") {
|
|
224
|
+
throw new Error(formatError(ctx, `stage "${edge.id}": fork.select must be "one" or "subset"${edge.fork.select === undefined ? " (missing)" : `, got "${String(edge.fork.select)}"`}`));
|
|
225
|
+
}
|
|
226
|
+
if ((dag.childrenOf[edge.id] ?? []).length === 0) {
|
|
227
|
+
throw new Error(formatError(ctx, `fork on stage "${edge.id}": no children in the DAG`));
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
export function resolvePipelineDagFromRefs(refs, ctx) {
|
|
232
|
+
const edges = normalizeToEdges(refs);
|
|
217
233
|
detectDuplicateIds(edges, ctx);
|
|
218
234
|
validateNeedsTargets(edges, ctx);
|
|
219
235
|
detectCycle(edges, ctx);
|
|
@@ -222,8 +238,13 @@ export function resolvePipelineDag(rawStages, ctx) {
|
|
|
222
238
|
.sort((a, b) => a.stageIndex - b.stageIndex)
|
|
223
239
|
.map((edge) => edge.id);
|
|
224
240
|
const dag = buildResolvedPipelineDag(edges);
|
|
241
|
+
validateForkFields(edges, dag, ctx);
|
|
225
242
|
return { stages, dag };
|
|
226
243
|
}
|
|
244
|
+
export function resolvePipelineDag(rawStages, ctx) {
|
|
245
|
+
const entries = parsePipelineStageEntries(rawStages, ctx);
|
|
246
|
+
return resolvePipelineDagFromRefs(entries, ctx);
|
|
247
|
+
}
|
|
227
248
|
export function areResolvedDagsEquivalent(a, b) {
|
|
228
249
|
const nodeIdsA = new Set(a.nodes.map((node) => node.id));
|
|
229
250
|
const nodeIdsB = new Set(b.nodes.map((node) => node.id));
|
|
@@ -256,16 +277,13 @@ export function extractPipelineStageIds(rawStages) {
|
|
|
256
277
|
const stageIds = [];
|
|
257
278
|
for (const entry of rawStages) {
|
|
258
279
|
if (typeof entry === "string") {
|
|
259
|
-
|
|
260
|
-
return null;
|
|
261
|
-
stageIds.push(entry);
|
|
262
|
-
continue;
|
|
280
|
+
return null;
|
|
263
281
|
}
|
|
264
282
|
if (!isPlainObject(entry))
|
|
265
283
|
return null;
|
|
266
284
|
const keys = Object.keys(entry);
|
|
267
285
|
for (const key of keys) {
|
|
268
|
-
if (!
|
|
286
|
+
if (!isAllowedPipelineStageEntryKey(key))
|
|
269
287
|
return null;
|
|
270
288
|
}
|
|
271
289
|
if (typeof entry.id !== "string" || !entry.id)
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { LoadedManifest } from "../types/stageflowManifest.js";
|
|
2
|
+
export type CatalogPathKind = "pipeline" | "task";
|
|
3
|
+
export declare function isExcluded(repoRelPath: string, exclude: string[] | undefined): boolean;
|
|
4
|
+
export declare function scanCatalogPaths(manifest: LoadedManifest, kind: CatalogPathKind): Promise<string[]>;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { readdir, stat } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
function normalizeRepoRel(repoRelPath) {
|
|
4
|
+
return repoRelPath.replace(/\\/g, "/").replace(/^\.\/+/, "");
|
|
5
|
+
}
|
|
6
|
+
export function isExcluded(repoRelPath, exclude) {
|
|
7
|
+
if (!exclude || exclude.length === 0) {
|
|
8
|
+
return false;
|
|
9
|
+
}
|
|
10
|
+
const normalized = normalizeRepoRel(repoRelPath);
|
|
11
|
+
for (const entry of exclude) {
|
|
12
|
+
const prefix = normalizeRepoRel(entry);
|
|
13
|
+
if (normalized === prefix || normalized.startsWith(`${prefix}/`)) {
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
function globPatternToRegExp(pattern) {
|
|
20
|
+
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
21
|
+
return new RegExp(`^${escaped}$`);
|
|
22
|
+
}
|
|
23
|
+
function matchesPattern(basename, pattern) {
|
|
24
|
+
return globPatternToRegExp(pattern).test(basename);
|
|
25
|
+
}
|
|
26
|
+
async function walkDirectory(absDir) {
|
|
27
|
+
const files = [];
|
|
28
|
+
let entries;
|
|
29
|
+
try {
|
|
30
|
+
entries = await readdir(absDir, { withFileTypes: true, recursive: true });
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return files;
|
|
34
|
+
}
|
|
35
|
+
for (const entry of entries) {
|
|
36
|
+
if (!entry.isFile())
|
|
37
|
+
continue;
|
|
38
|
+
const parentDir = entry.parentPath ?? absDir;
|
|
39
|
+
files.push(path.join(parentDir, entry.name));
|
|
40
|
+
}
|
|
41
|
+
return files;
|
|
42
|
+
}
|
|
43
|
+
async function expandCatalogEntry(projectRoot, entry, kind, pattern) {
|
|
44
|
+
const absPath = path.join(projectRoot, entry);
|
|
45
|
+
let entryStat;
|
|
46
|
+
try {
|
|
47
|
+
entryStat = await stat(absPath);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return [];
|
|
51
|
+
}
|
|
52
|
+
if (entryStat.isFile()) {
|
|
53
|
+
return [absPath];
|
|
54
|
+
}
|
|
55
|
+
if (!entryStat.isDirectory()) {
|
|
56
|
+
return [];
|
|
57
|
+
}
|
|
58
|
+
const files = await walkDirectory(absPath);
|
|
59
|
+
return files.filter((filePath) => matchesPattern(path.basename(filePath), pattern));
|
|
60
|
+
}
|
|
61
|
+
export async function scanCatalogPaths(manifest, kind) {
|
|
62
|
+
const { projectRoot, manifest: doc, patterns } = manifest;
|
|
63
|
+
const entries = kind === "pipeline" ? doc.catalog.pipelines : doc.catalog.tasks;
|
|
64
|
+
const pattern = kind === "pipeline" ? patterns.pipeline : patterns.task;
|
|
65
|
+
const exclude = doc.catalog.exclude;
|
|
66
|
+
const seen = new Set();
|
|
67
|
+
const resolved = [];
|
|
68
|
+
for (const entry of entries) {
|
|
69
|
+
const absPaths = await expandCatalogEntry(projectRoot, entry, kind, pattern);
|
|
70
|
+
for (const absPath of absPaths) {
|
|
71
|
+
const normalized = path.resolve(absPath);
|
|
72
|
+
if (seen.has(normalized))
|
|
73
|
+
continue;
|
|
74
|
+
const repoRel = normalizeRepoRel(path.relative(projectRoot, normalized));
|
|
75
|
+
if (isExcluded(repoRel, exclude))
|
|
76
|
+
continue;
|
|
77
|
+
seen.add(normalized);
|
|
78
|
+
resolved.push(normalized);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
resolved.sort((a, b) => a.localeCompare(b));
|
|
82
|
+
return resolved;
|
|
83
|
+
}
|