tubeless 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -52
- package/dist/cli/cli-types.d.ts +1 -1
- package/dist/cli/cli-types.js.map +1 -1
- package/dist/cli/cli.d.ts +2 -4
- package/dist/cli/cli.js +1 -3
- package/dist/cli/cli.js.map +1 -1
- package/dist/core/child-execution.js +9 -4
- package/dist/core/child-execution.js.map +1 -1
- package/dist/core/lifecycle.d.ts +1 -1
- package/dist/core/lifecycle.js +2 -0
- package/dist/core/lifecycle.js.map +1 -1
- package/dist/core/pipeline-compiler.d.ts +17 -0
- package/dist/core/pipeline-compiler.js +44 -0
- package/dist/core/pipeline-compiler.js.map +1 -0
- package/dist/core/pipeline-definition-validation.d.ts +4 -0
- package/dist/core/pipeline-definition-validation.js +101 -0
- package/dist/core/pipeline-definition-validation.js.map +1 -0
- package/dist/core/pipeline-definition.d.ts +28 -0
- package/dist/core/pipeline-definition.js +2 -0
- package/dist/core/pipeline-definition.js.map +1 -0
- package/dist/core/pipeline-disposition.d.ts +26 -0
- package/dist/core/pipeline-disposition.js +44 -0
- package/dist/core/pipeline-disposition.js.map +1 -0
- package/dist/core/pipeline-errors.d.ts +8 -0
- package/dist/core/pipeline-errors.js +20 -0
- package/dist/core/pipeline-errors.js.map +1 -0
- package/dist/core/pipeline-execute.d.ts +3 -1
- package/dist/core/pipeline-execute.js +136 -396
- package/dist/core/pipeline-execute.js.map +1 -1
- package/dist/core/pipeline-finalizer.d.ts +18 -0
- package/dist/core/pipeline-finalizer.js +37 -0
- package/dist/core/pipeline-finalizer.js.map +1 -0
- package/dist/core/pipeline-graph.d.ts +23 -0
- package/dist/core/pipeline-graph.js +135 -0
- package/dist/core/pipeline-graph.js.map +1 -0
- package/dist/core/pipeline-identity.d.ts +5 -0
- package/dist/core/pipeline-identity.js +11 -0
- package/dist/core/pipeline-identity.js.map +1 -0
- package/dist/core/pipeline-mermaid.d.ts +4 -0
- package/dist/core/pipeline-mermaid.js +56 -0
- package/dist/core/pipeline-mermaid.js.map +1 -0
- package/dist/core/pipeline-plan.d.ts +5 -94
- package/dist/core/pipeline-plan.js +5 -392
- package/dist/core/pipeline-plan.js.map +1 -1
- package/dist/core/pipeline-run-state.d.ts +49 -0
- package/dist/core/pipeline-run-state.js +284 -0
- package/dist/core/pipeline-run-state.js.map +1 -0
- package/dist/core/pipeline-step-executor.d.ts +20 -0
- package/dist/core/pipeline-step-executor.js +52 -0
- package/dist/core/pipeline-step-executor.js.map +1 -0
- package/dist/core/pipeline-step-metadata.d.ts +4 -0
- package/dist/core/pipeline-step-metadata.js +5 -0
- package/dist/core/pipeline-step-metadata.js.map +1 -0
- package/dist/core/pipeline-steps.d.ts +1 -1
- package/dist/core/pipeline-steps.js +1 -1
- package/dist/core/pipeline-steps.js.map +1 -1
- package/dist/core/pipeline-types.d.ts +17 -8
- package/dist/core/pipeline-types.js +8 -0
- package/dist/core/pipeline-types.js.map +1 -1
- package/dist/core/pipeline-validation.d.ts +6 -0
- package/dist/core/pipeline-validation.js +41 -0
- package/dist/core/pipeline-validation.js.map +1 -0
- package/dist/core/pipeline.d.ts +8 -9
- package/dist/core/pipeline.js +9 -6
- package/dist/core/pipeline.js.map +1 -1
- package/dist/node/node.d.ts +0 -3
- package/dist/node/node.js +0 -3
- package/dist/node/node.js.map +1 -1
- package/dist/reporter/reporter-entry.d.ts +1 -2
- package/dist/reporter/reporter-entry.js +1 -2
- package/dist/reporter/reporter-entry.js.map +1 -1
- package/dist/run-store/run-store-codec.d.ts +4 -0
- package/dist/run-store/run-store-codec.js +31 -0
- package/dist/run-store/run-store-codec.js.map +1 -0
- package/dist/run-store/run-store-ndjson.js +2 -295
- package/dist/run-store/run-store-ndjson.js.map +1 -1
- package/dist/run-store/run-store-sqlite.js +41 -38
- package/dist/run-store/run-store-sqlite.js.map +1 -1
- package/dist/run-store/run-store.d.ts +4 -3
- package/dist/run-store/run-store.js +38 -195
- package/dist/run-store/run-store.js.map +1 -1
- package/dist/studio/run-store-ui-client-app.d.ts +12 -0
- package/dist/studio/run-store-ui-client-app.js +224 -0
- package/dist/studio/run-store-ui-client-app.js.map +1 -0
- package/dist/studio/run-store-ui-client-browser.d.ts +1 -0
- package/dist/studio/run-store-ui-client-browser.js +2 -0
- package/dist/studio/run-store-ui-client-commands.d.ts +18 -0
- package/dist/studio/run-store-ui-client-commands.js +98 -0
- package/dist/studio/run-store-ui-client-commands.js.map +1 -0
- package/dist/studio/run-store-ui-client-form.d.ts +11 -0
- package/dist/studio/run-store-ui-client-form.js +54 -0
- package/dist/studio/run-store-ui-client-form.js.map +1 -0
- package/dist/studio/run-store-ui-client-modals.d.ts +20 -0
- package/dist/studio/run-store-ui-client-modals.js +154 -0
- package/dist/studio/run-store-ui-client-modals.js.map +1 -0
- package/dist/studio/run-store-ui-client-model.d.ts +19 -0
- package/dist/studio/run-store-ui-client-model.js +189 -0
- package/dist/studio/run-store-ui-client-model.js.map +1 -0
- package/dist/studio/run-store-ui-client-runs.d.ts +24 -0
- package/dist/studio/run-store-ui-client-runs.js +78 -0
- package/dist/studio/run-store-ui-client-runs.js.map +1 -0
- package/dist/studio/run-store-ui-client-shared.d.ts +22 -0
- package/dist/studio/run-store-ui-client-shared.js +63 -0
- package/dist/studio/run-store-ui-client-shared.js.map +1 -0
- package/dist/studio/run-store-ui-client-transport.d.ts +25 -0
- package/dist/studio/run-store-ui-client-transport.js +317 -0
- package/dist/studio/run-store-ui-client-transport.js.map +1 -0
- package/dist/studio/run-store-ui-client.d.ts +2 -14
- package/dist/studio/run-store-ui-client.js +9 -1360
- package/dist/studio/run-store-ui-client.js.map +1 -1
- package/dist/studio/run-store-ui-page.d.ts +2 -2
- package/dist/studio/run-store-ui-page.js +7 -71
- package/dist/studio/run-store-ui-page.js.map +1 -1
- package/dist/studio/run-store-ui.d.ts +1 -1
- package/dist/studio/run-store-ui.js +43 -25
- package/dist/studio/run-store-ui.js.map +1 -1
- package/dist/tracing/tracing-codec.d.ts +5 -0
- package/dist/tracing/tracing-codec.js +13 -0
- package/dist/tracing/tracing-codec.js.map +1 -0
- package/dist/tracing/tracing-constants.d.ts +4 -0
- package/dist/tracing/tracing-constants.js +5 -0
- package/dist/tracing/tracing-constants.js.map +1 -0
- package/dist/tracing/tracing-contracts.d.ts +31 -0
- package/dist/tracing/tracing-contracts.js +2 -0
- package/dist/tracing/tracing-contracts.js.map +1 -0
- package/dist/tracing/tracing-internal.d.ts +1 -1
- package/dist/tracing/tracing-internal.js +116 -81
- package/dist/tracing/tracing-internal.js.map +1 -1
- package/dist/tracing/tracing-schema.d.ts +809 -0
- package/dist/tracing/tracing-schema.js +429 -0
- package/dist/tracing/tracing-schema.js.map +1 -0
- package/dist/tracing/tracing.d.ts +2 -54
- package/dist/tracing/tracing.js.map +1 -1
- package/dist/tracing/wire-schema.d.ts +50 -0
- package/dist/tracing/wire-schema.js +193 -0
- package/dist/tracing/wire-schema.js.map +1 -0
- package/dist/workbench/pipeline-module.js +4 -4
- package/dist/workbench/pipeline-module.js.map +1 -1
- package/dist/workbench/workbench-entry.d.ts +2 -0
- package/dist/workbench/workbench-entry.js +3 -0
- package/dist/workbench/workbench-entry.js.map +1 -0
- package/dist/workbench/workbench-history.js +8 -1
- package/dist/workbench/workbench-history.js.map +1 -1
- package/dist/workbench/workbench-ui.js +39 -43
- package/dist/workbench/workbench-ui.js.map +1 -1
- package/docs/README.md +22 -15
- package/docs/agent-guide.md +62 -68
- package/docs/agent-skills.md +59 -0
- package/docs/api-reference.md +14 -178
- package/docs/api-report.json +13 -215
- package/docs/child-pipeline-composition.md +139 -193
- package/docs/cli.md +86 -64
- package/docs/comparison.md +64 -62
- package/docs/concepts.md +210 -281
- package/docs/getting-started.md +61 -49
- package/docs/llms.txt +8 -8
- package/docs/recipes.md +42 -45
- package/docs/remote-step-composition.md +69 -74
- package/docs/studio.md +110 -73
- package/examples/catalog/pipelines/enrich.ts +2 -2
- package/examples/catalog/scripts/enrich.ts +1 -1
- package/examples/catalog/scripts/import.ts +1 -1
- package/examples/catalog/scripts/publish.ts +1 -1
- package/examples/catalog/tubeless.project.ts +6 -3
- package/examples/cli-job.ts +1 -1
- package/examples/host-embedding.ts +4 -5
- package/examples/live-tui.ts +1 -1
- package/examples/peloton.ts +1 -1
- package/examples/remote-steps.ts +8 -5
- package/examples/tracing.ts +96 -5
- package/package.json +17 -64
- package/skills/tubeless/SKILL.md +85 -0
- package/skills/tubeless-make-pipeline/SKILL.md +126 -0
- package/dist/node/env.d.ts +0 -1
- package/dist/node/env.js +0 -8
- package/dist/node/env.js.map +0 -1
- package/dist/node/file-utils.d.ts +0 -3
- package/dist/node/file-utils.js +0 -17
- package/dist/node/file-utils.js.map +0 -1
- package/dist/node/paths.d.ts +0 -8
- package/dist/node/paths.js +0 -19
- package/dist/node/paths.js.map +0 -1
- package/dist/reporter/prompt-select.d.ts +0 -80
- package/dist/reporter/prompt-select.js +0 -197
- package/dist/reporter/prompt-select.js.map +0 -1
- package/dist/studio/run-store-ui-client-source.d.ts +0 -1
- package/dist/studio/run-store-ui-client-source.js +0 -1
- package/dist/tracing/tracing-json.d.ts +0 -11
- package/dist/tracing/tracing-json.js +0 -8
- package/dist/tracing/tracing-json.js.map +0 -1
- package/dist/tracing/tracing-otel.d.ts +0 -35
- package/dist/tracing/tracing-otel.js +0 -60
- package/dist/tracing/tracing-otel.js.map +0 -1
- package/dist/workbench/workbench-studio.d.ts +0 -29
- package/dist/workbench/workbench-studio.js +0 -61
- package/dist/workbench/workbench-studio.js.map +0 -1
- package/examples/catalog/tubeless.studio.ts +0 -20
- package/examples/local-observability.ts +0 -105
- package/examples/rendering.ts +0 -34
|
@@ -3,16 +3,21 @@ import { PipelineChildError } from "./child-execution.js";
|
|
|
3
3
|
import { brandTubelessError } from "../utilities/tubeless-error.js";
|
|
4
4
|
import { createPipelineLifecycleObserver } from "./lifecycle.js";
|
|
5
5
|
import { formatPipelineError } from "./pipeline-diagnostics.js";
|
|
6
|
-
import { createRunId
|
|
7
|
-
import {
|
|
6
|
+
import { createRunId } from "./pipeline-ids.js";
|
|
7
|
+
import { decideStepDisposition } from "./pipeline-disposition.js";
|
|
8
|
+
import { compiledStepGraph } from "./pipeline-graph.js";
|
|
9
|
+
import { planStepById, stepToPlanStep } from "./pipeline-plan.js";
|
|
10
|
+
import { PipelineRunState, isCancellationOnly, } from "./pipeline-run-state.js";
|
|
11
|
+
import { PIPELINE_FINALIZE_STEP_ID } from "./pipeline-step-metadata.js";
|
|
12
|
+
import { evaluateStepSkip, executeStepAttempt, validateStepOutput, } from "./pipeline-step-executor.js";
|
|
13
|
+
import { PipelineBoundaryValidationError, validateStandardSchema } from "./pipeline-validation.js";
|
|
8
14
|
import { createPipelineTraceEmitter } from "../tracing/tracing-internal.js";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
return isCancellationOnly(errors) ? "cancelled" : "failed";
|
|
15
|
+
const PIPELINE_RUN_ID_OBSERVER = Symbol.for("tubeless.pipeline.runIdObserver");
|
|
16
|
+
/** Observe an execution ID from an internal host without allowing it to be overridden. */
|
|
17
|
+
export function observePipelineRunId(context, observer) {
|
|
18
|
+
// SAFETY: the internal observer is intentionally hidden from the returned
|
|
19
|
+
// public type and can observe, but never choose, the generated run ID.
|
|
20
|
+
return { ...context, [PIPELINE_RUN_ID_OBSERVER]: observer };
|
|
16
21
|
}
|
|
17
22
|
const PIPELINE_LOGGER_BASE = Symbol("pipelineLoggerBase");
|
|
18
23
|
function basePipelineLogger(log) {
|
|
@@ -46,46 +51,6 @@ export class PipelineExecutionError extends Error {
|
|
|
46
51
|
brandTubelessError(this, "pipeline-execution");
|
|
47
52
|
}
|
|
48
53
|
}
|
|
49
|
-
class PipelineBoundaryValidationError extends Error {
|
|
50
|
-
issues;
|
|
51
|
-
constructor(message, issues, cause) {
|
|
52
|
-
super(message, cause === undefined ? undefined : { cause });
|
|
53
|
-
this.issues = issues;
|
|
54
|
-
this.name = "PipelineBoundaryValidationError";
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
function normalizeValidationPath(path) {
|
|
58
|
-
if (!path || path.length === 0)
|
|
59
|
-
return undefined;
|
|
60
|
-
return path.map((segment) => {
|
|
61
|
-
const key = typeof segment === "object" && segment !== null ? segment.key : segment;
|
|
62
|
-
return typeof key === "number" || typeof key === "string" ? key : String(key);
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
function renderValidationIssue(issue) {
|
|
66
|
-
const location = issue.path?.map(String).join(".");
|
|
67
|
-
return location ? `${location}: ${issue.message}` : issue.message;
|
|
68
|
-
}
|
|
69
|
-
async function validateStandardSchema(schema, value, boundary) {
|
|
70
|
-
let result;
|
|
71
|
-
try {
|
|
72
|
-
result = await schema["~standard"].validate(value);
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
throw new PipelineBoundaryValidationError(`${boundary} schema (${schema["~standard"].vendor}) threw while validating`, [], error);
|
|
76
|
-
}
|
|
77
|
-
if (result.issues) {
|
|
78
|
-
const issues = result.issues.map((issue) => {
|
|
79
|
-
const path = normalizeValidationPath(issue.path);
|
|
80
|
-
const normalized = { message: issue.message };
|
|
81
|
-
if (path)
|
|
82
|
-
normalized.path = path;
|
|
83
|
-
return normalized;
|
|
84
|
-
});
|
|
85
|
-
throw new PipelineBoundaryValidationError(`${boundary} validation failed: ${issues.map(renderValidationIssue).join("; ")}`, issues);
|
|
86
|
-
}
|
|
87
|
-
return result.value;
|
|
88
|
-
}
|
|
89
54
|
const MAX_PIPELINE_CAUSE_DEPTH = 8;
|
|
90
55
|
function readErrorField(value, field) {
|
|
91
56
|
try {
|
|
@@ -201,81 +166,29 @@ export function isPipelineCancellation(error, runtime) {
|
|
|
201
166
|
function isCancelledResult(result) {
|
|
202
167
|
return result.status === "cancelled";
|
|
203
168
|
}
|
|
204
|
-
function errorRunResult(runtime, pipelineId, dryRun, startedAt, errors, identity) {
|
|
205
|
-
const finishedAtMs = runtime.now();
|
|
206
|
-
const result = {
|
|
207
|
-
pipelineId,
|
|
208
|
-
dryRun,
|
|
209
|
-
errors,
|
|
210
|
-
finalized: false,
|
|
211
|
-
finishedAtMs,
|
|
212
|
-
runId: identity.runId,
|
|
213
|
-
startedAtMs: startedAt,
|
|
214
|
-
status: terminalRunStatus(errors),
|
|
215
|
-
steps: [],
|
|
216
|
-
version: RUN_MODEL_VERSION,
|
|
217
|
-
};
|
|
218
|
-
if (identity.parentRunId)
|
|
219
|
-
result.parentRunId = identity.parentRunId;
|
|
220
|
-
return result;
|
|
221
|
-
}
|
|
222
169
|
function throwIfAborted(runtime) {
|
|
223
170
|
throwIfSignalAborted(runtime.signal, "Pipeline run");
|
|
224
171
|
}
|
|
225
|
-
function recordSkip(step, stepId, reports, reportsByStepId, reason, message, dependencyId, attemptId, startedAtMs, finishedAtMs) {
|
|
226
|
-
const report = {
|
|
227
|
-
id: stepId,
|
|
228
|
-
name: step.name,
|
|
229
|
-
description: step.description,
|
|
230
|
-
finishedAtMs,
|
|
231
|
-
reason,
|
|
232
|
-
status: "skipped",
|
|
233
|
-
};
|
|
234
|
-
if (attemptId)
|
|
235
|
-
report.attemptId = attemptId;
|
|
236
|
-
if (startedAtMs !== undefined)
|
|
237
|
-
report.startedAtMs = startedAtMs;
|
|
238
|
-
if (dependencyId)
|
|
239
|
-
report.dependencyId = dependencyId;
|
|
240
|
-
if (message)
|
|
241
|
-
report.message = message;
|
|
242
|
-
reports.push(report);
|
|
243
|
-
reportsByStepId.set(stepId, report);
|
|
244
|
-
return report;
|
|
245
|
-
}
|
|
246
|
-
function normalizeStepSkipDecision(decision) {
|
|
247
|
-
if (decision == null || decision === false)
|
|
248
|
-
return null;
|
|
249
|
-
if (typeof decision === "string") {
|
|
250
|
-
const reason = decision.trim();
|
|
251
|
-
return reason.length > 0 ? { reason } : null;
|
|
252
|
-
}
|
|
253
|
-
if (typeof decision === "object" && typeof decision.reason === "string") {
|
|
254
|
-
const reason = decision.reason.trim();
|
|
255
|
-
if (reason.length === 0)
|
|
256
|
-
return null;
|
|
257
|
-
return { reason, value: decision.value };
|
|
258
|
-
}
|
|
259
|
-
return null;
|
|
260
|
-
}
|
|
261
172
|
export async function executePlannedRun(input) {
|
|
262
173
|
const { compiled, controls, runtime } = input;
|
|
263
|
-
const
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
174
|
+
const startedAtMs = runtime.now();
|
|
175
|
+
const runId = createRunId(compiled.id);
|
|
176
|
+
try {
|
|
177
|
+
// SAFETY: only internal hosts attach this observation callback. A callback
|
|
178
|
+
// failure must not change whether the pipeline execution itself can start.
|
|
179
|
+
runtime[PIPELINE_RUN_ID_OBSERVER]?.(runId);
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
// Ignore observation failures; execution identity remains package-owned.
|
|
183
|
+
}
|
|
184
|
+
const correlationId = runtime.correlationId;
|
|
271
185
|
const identity = { runId };
|
|
272
|
-
if (
|
|
273
|
-
identity.
|
|
186
|
+
if (correlationId !== undefined)
|
|
187
|
+
identity.correlationId = correlationId;
|
|
188
|
+
if (runtime.parentRunId)
|
|
189
|
+
identity.parentRunId = runtime.parentRunId;
|
|
274
190
|
const dryRun = controls.dryRun === true;
|
|
275
|
-
const trace = createPipelineTraceEmitter(compiled.id, runtime.tracing, basePipelineLogger(runtime.log), {
|
|
276
|
-
...identity,
|
|
277
|
-
itemKey: runtime.tracing?.itemKey,
|
|
278
|
-
}, runtime.now);
|
|
191
|
+
const trace = createPipelineTraceEmitter(compiled.id, runtime.tracing, basePipelineLogger(runtime.log), { ...identity, itemKey: runtime.tracing?.itemKey }, runtime.now);
|
|
279
192
|
const lifecycle = createPipelineLifecycleObserver(compiled.id, runtime, trace);
|
|
280
193
|
const tracedLogger = (stepId, attemptId) => {
|
|
281
194
|
if (!trace)
|
|
@@ -298,53 +211,18 @@ export async function executePlannedRun(input) {
|
|
|
298
211
|
log[PIPELINE_LOGGER_BASE] = base;
|
|
299
212
|
return log;
|
|
300
213
|
};
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
});
|
|
307
|
-
const currentStepStatuses = new Map();
|
|
308
|
-
const publishStepStatus = (event, traceStatus = true) => {
|
|
309
|
-
const previous = currentStepStatuses.get(event.step.id);
|
|
310
|
-
const valid = (previous === undefined && event.status === "planned") ||
|
|
311
|
-
(previous === "planned" &&
|
|
312
|
-
(event.status === "running" ||
|
|
313
|
-
event.status === "skipped" ||
|
|
314
|
-
event.status === "cancelled")) ||
|
|
315
|
-
(previous === "running" &&
|
|
316
|
-
(event.status === "running" ||
|
|
317
|
-
event.status === "completed" ||
|
|
318
|
-
event.status === "skipped" ||
|
|
319
|
-
event.status === "cancelled" ||
|
|
320
|
-
event.status === "failed"));
|
|
321
|
-
if (!valid) {
|
|
322
|
-
throw new Error(`Invalid step status transition for ${event.step.id}: ${previous ?? "unobserved"} -> ${event.status}`);
|
|
323
|
-
}
|
|
324
|
-
currentStepStatuses.set(event.step.id, event.status);
|
|
325
|
-
lifecycle.stepStatus(event, traceStatus);
|
|
326
|
-
};
|
|
327
|
-
const publishStepReport = (step, report) => {
|
|
328
|
-
switch (report.status) {
|
|
329
|
-
case "cancelled":
|
|
330
|
-
case "failed":
|
|
331
|
-
case "skipped":
|
|
332
|
-
case "completed":
|
|
333
|
-
publishStepStatus({ ...report, pipelineId: compiled.id, step });
|
|
334
|
-
}
|
|
335
|
-
};
|
|
336
|
-
const runPlan = input.plan;
|
|
337
|
-
const plannedSteps = planStepById(runPlan);
|
|
338
|
-
lifecycle.pipelineStart(runPlan, targetIds);
|
|
339
|
-
if (!runPlan.ok) {
|
|
340
|
-
const result = errorRunResult(runtime, compiled.id, dryRun, startedAt, runPlan.errors, identity);
|
|
341
|
-
lifecycle.pipelineComplete(result);
|
|
342
|
-
await lifecycle.flush();
|
|
343
|
-
return result;
|
|
214
|
+
const state = new PipelineRunState(compiled.id, dryRun, startedAtMs, identity, runtime.now, lifecycle);
|
|
215
|
+
state.start(input.plan, compiled.targetIds);
|
|
216
|
+
if (!input.plan.ok) {
|
|
217
|
+
state.recordRunErrors(input.plan.errors);
|
|
218
|
+
return state.finish();
|
|
344
219
|
}
|
|
345
|
-
if
|
|
220
|
+
// SAFETY: `domainOptions` is the user-supplied options object; if a schema
|
|
221
|
+
// is present it is re-validated below before assignment to `pipelineOptions`.
|
|
222
|
+
let pipelineOptions = input.domainOptions;
|
|
223
|
+
if (compiled.optionsSchema) {
|
|
346
224
|
try {
|
|
347
|
-
const validated = await validateStandardSchema(optionsSchema, input.domainOptions, `Pipeline ${compiled.id} options`);
|
|
225
|
+
const validated = await validateStandardSchema(compiled.optionsSchema, input.domainOptions, `Pipeline ${compiled.id} options`);
|
|
348
226
|
if (typeof validated !== "object" || validated === null || Array.isArray(validated)) {
|
|
349
227
|
throw new PipelineBoundaryValidationError(`Pipeline ${compiled.id} options schema returned a non-object value`, [{ message: "Expected the validated options value to be an object" }]);
|
|
350
228
|
}
|
|
@@ -353,25 +231,18 @@ export async function executePlannedRun(input) {
|
|
|
353
231
|
pipelineOptions = validated;
|
|
354
232
|
}
|
|
355
233
|
catch (error) {
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
return result;
|
|
234
|
+
state.recordRunErrors([
|
|
235
|
+
toPipelineError(error, {
|
|
236
|
+
code: "TUBELESS_OPTIONS_VALIDATION_FAILED",
|
|
237
|
+
kind: "validation",
|
|
238
|
+
phase: "execution",
|
|
239
|
+
}),
|
|
240
|
+
]);
|
|
241
|
+
return state.finish();
|
|
365
242
|
}
|
|
366
243
|
}
|
|
367
|
-
for (const step of
|
|
368
|
-
|
|
369
|
-
}
|
|
370
|
-
const outputs = new Map();
|
|
371
|
-
const reports = [];
|
|
372
|
-
const errors = [];
|
|
373
|
-
const reportsByStepId = new Map();
|
|
374
|
-
let completed = true;
|
|
244
|
+
for (const step of input.plan.steps)
|
|
245
|
+
state.planStep(step);
|
|
375
246
|
const executionContext = {
|
|
376
247
|
...runtime,
|
|
377
248
|
dryRun,
|
|
@@ -380,50 +251,50 @@ export async function executePlannedRun(input) {
|
|
|
380
251
|
runId,
|
|
381
252
|
trace: trace?.context,
|
|
382
253
|
};
|
|
383
|
-
if (
|
|
384
|
-
executionContext.
|
|
254
|
+
if (correlationId !== undefined)
|
|
255
|
+
executionContext.correlationId = correlationId;
|
|
256
|
+
if (runtime.parentRunId)
|
|
257
|
+
executionContext.parentRunId = runtime.parentRunId;
|
|
258
|
+
const plannedSteps = planStepById(input.plan);
|
|
385
259
|
const stepsById = new Map(compiled.orderedSteps.map((step) => [step.id, step]));
|
|
386
|
-
// `
|
|
387
|
-
// objects so callers still get the typed
|
|
260
|
+
// `input.plan.steps` is already topologically ordered. Rehydrate from the original step
|
|
261
|
+
// objects so callers still get the typed handlers rather than plan metadata.
|
|
388
262
|
// SAFETY: every planned step id originates from `compiled.orderedSteps`, so the
|
|
389
263
|
// map lookup always finds the matching `AnyStep<TOptions>`.
|
|
390
|
-
const orderedSteps =
|
|
264
|
+
const orderedSteps = input.plan.steps.map((step) => stepsById.get(step.id));
|
|
265
|
+
const plannedStepFor = (step) => plannedSteps.get(step.id) ??
|
|
266
|
+
stepToPlanStep(step, true, undefined, undefined, compiledStepGraph(compiled, step));
|
|
391
267
|
const recordRemainingStates = (fromIndex, failedStepId, cancellationError) => {
|
|
392
268
|
for (let index = fromIndex; index < orderedSteps.length; index++) {
|
|
393
269
|
const step = orderedSteps[index];
|
|
394
|
-
const plannedStep =
|
|
395
|
-
stepToPlanStep(step, true, undefined, undefined, compiledStepGraph(compiled, step));
|
|
270
|
+
const plannedStep = plannedStepFor(step);
|
|
396
271
|
if (plannedStep.skipReason) {
|
|
397
272
|
const dependencyId = plannedStep.skipReason === "unmet-dependency"
|
|
398
273
|
? plannedStep.dependencies.find((id) => plannedSteps.get(id)?.skipReason !== undefined)
|
|
399
274
|
: undefined;
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
275
|
+
state.skipStep(plannedStep, {
|
|
276
|
+
reason: plannedStep.skipReason,
|
|
277
|
+
dependencyId,
|
|
278
|
+
});
|
|
403
279
|
}
|
|
404
|
-
if (cancellationError) {
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
publishStepReport(plannedStep, report);
|
|
416
|
-
continue;
|
|
280
|
+
else if (cancellationError) {
|
|
281
|
+
state.cancelStep(plannedStep, { ...cancellationError, stepId: step.id }, false);
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
state.skipStep(plannedStep, {
|
|
285
|
+
reason: "fail-fast",
|
|
286
|
+
message: failedStepId
|
|
287
|
+
? `Not run because fail-fast stopped after ${failedStepId} failed.`
|
|
288
|
+
: "Not run because the pipeline was aborted before this step started.",
|
|
289
|
+
dependencyId: failedStepId,
|
|
290
|
+
});
|
|
417
291
|
}
|
|
418
|
-
const report = recordSkip(step, step.id, reports, reportsByStepId, "fail-fast", failedStepId
|
|
419
|
-
? `Not run because fail-fast stopped after ${failedStepId} failed.`
|
|
420
|
-
: "Not run because the pipeline was aborted before this step started.", failedStepId, undefined, undefined, runtime.now());
|
|
421
|
-
publishStepReport(plannedStep, report);
|
|
422
292
|
}
|
|
423
293
|
};
|
|
424
294
|
const cancelBeforeStepStart = (stepId, stepIndex) => {
|
|
425
295
|
try {
|
|
426
296
|
throwIfAborted(runtime);
|
|
297
|
+
return false;
|
|
427
298
|
}
|
|
428
299
|
catch (error) {
|
|
429
300
|
const pipelineError = toPipelineError(error, {
|
|
@@ -432,14 +303,12 @@ export async function executePlannedRun(input) {
|
|
|
432
303
|
phase: "execution",
|
|
433
304
|
stepId,
|
|
434
305
|
});
|
|
435
|
-
|
|
436
|
-
completed = false;
|
|
306
|
+
state.recordRunErrors([pipelineError]);
|
|
437
307
|
recordRemainingStates(stepIndex, undefined, pipelineError);
|
|
438
308
|
return true;
|
|
439
309
|
}
|
|
440
|
-
return false;
|
|
441
310
|
};
|
|
442
|
-
const recordStepExecutionFailure = (error, plannedStep,
|
|
311
|
+
const recordStepExecutionFailure = (error, plannedStep, stepIndex, attempt) => {
|
|
443
312
|
const cancelled = isPipelineCancellation(error, runtime);
|
|
444
313
|
const childFailure = error instanceof PipelineExecutionError || error instanceof PipelineChildError;
|
|
445
314
|
const validationFailure = error instanceof PipelineBoundaryValidationError;
|
|
@@ -459,217 +328,109 @@ export async function executePlannedRun(input) {
|
|
|
459
328
|
? "child"
|
|
460
329
|
: "step",
|
|
461
330
|
phase: "execution",
|
|
462
|
-
stepId,
|
|
331
|
+
stepId: plannedStep.id,
|
|
463
332
|
});
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
id: stepId,
|
|
470
|
-
name: step.name,
|
|
471
|
-
description: step.description,
|
|
472
|
-
error: pipelineError,
|
|
473
|
-
finishedAtMs,
|
|
474
|
-
startedAtMs: attempt.startedAtMs,
|
|
475
|
-
status: "cancelled",
|
|
476
|
-
}
|
|
477
|
-
: {
|
|
478
|
-
attemptId: attempt.attemptId,
|
|
479
|
-
id: stepId,
|
|
480
|
-
name: step.name,
|
|
481
|
-
description: step.description,
|
|
482
|
-
error: pipelineError,
|
|
483
|
-
finishedAtMs,
|
|
484
|
-
startedAtMs: attempt.startedAtMs,
|
|
485
|
-
status: "failed",
|
|
486
|
-
};
|
|
487
|
-
reports.push(report);
|
|
488
|
-
reportsByStepId.set(stepId, report);
|
|
489
|
-
publishStepReport(plannedStep, report);
|
|
490
|
-
completed = false;
|
|
491
|
-
if (!controls.continueOnError) {
|
|
492
|
-
recordRemainingStates(stepIndex + 1, stepId, report.status === "cancelled" ? pipelineError : undefined);
|
|
493
|
-
return true;
|
|
494
|
-
}
|
|
495
|
-
return false;
|
|
333
|
+
state.failStep(plannedStep, attempt, pipelineError);
|
|
334
|
+
if (controls.continueOnError)
|
|
335
|
+
return false;
|
|
336
|
+
recordRemainingStates(stepIndex + 1, plannedStep.id, cancelled ? pipelineError : undefined);
|
|
337
|
+
return true;
|
|
496
338
|
};
|
|
497
|
-
const
|
|
498
|
-
let
|
|
499
|
-
let
|
|
500
|
-
if (
|
|
501
|
-
|
|
502
|
-
attempt = beginAttempt(args.stepId, runtime.now());
|
|
503
|
-
publishStepStatus({
|
|
504
|
-
attemptId: attempt.attemptId,
|
|
505
|
-
pipelineId: compiled.id,
|
|
506
|
-
status: "running",
|
|
507
|
-
step: args.plannedStep,
|
|
508
|
-
});
|
|
509
|
-
}
|
|
339
|
+
const recordPolicySkip = async (plannedStep, step, stepIndex, reason, value) => {
|
|
340
|
+
let attempt;
|
|
341
|
+
let published = value;
|
|
342
|
+
if (step.outputSchema) {
|
|
343
|
+
attempt = state.beginAttempt(plannedStep);
|
|
510
344
|
try {
|
|
511
|
-
published = await
|
|
345
|
+
published = await validateStepOutput(step, value, `Pipeline ${compiled.id} step ${step.id} output`);
|
|
512
346
|
}
|
|
513
347
|
catch (error) {
|
|
514
|
-
return recordStepExecutionFailure(error,
|
|
348
|
+
return recordStepExecutionFailure(error, plannedStep, stepIndex, attempt);
|
|
515
349
|
}
|
|
516
350
|
}
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
name: args.step.name,
|
|
524
|
-
description: args.step.description,
|
|
525
|
-
finishedAtMs: runtime.now(),
|
|
526
|
-
startedAtMs: completeAttempt.startedAtMs,
|
|
527
|
-
status: "completed",
|
|
528
|
-
};
|
|
529
|
-
reports.push(report);
|
|
530
|
-
reportsByStepId.set(args.stepId, report);
|
|
531
|
-
publishStepReport(args.plannedStep, report);
|
|
532
|
-
return false;
|
|
533
|
-
}
|
|
534
|
-
const report = recordSkip(args.step, args.stepId, reports, reportsByStepId, "policy", args.skipMessage, undefined, attempt?.attemptId, attempt?.startedAtMs, runtime.now());
|
|
535
|
-
publishStepReport(args.plannedStep, report);
|
|
351
|
+
state.skipStep(plannedStep, {
|
|
352
|
+
attempt,
|
|
353
|
+
message: reason,
|
|
354
|
+
output: { value: published },
|
|
355
|
+
reason: "policy",
|
|
356
|
+
});
|
|
536
357
|
return false;
|
|
537
358
|
};
|
|
538
359
|
for (const [stepIndex, step] of orderedSteps.entries()) {
|
|
539
|
-
const
|
|
540
|
-
|
|
541
|
-
const plannedStep = plannedSteps.get(stepId) ?? stepToPlanStep(step, true, undefined, undefined, graph);
|
|
542
|
-
if (cancelBeforeStepStart(stepId, stepIndex)) {
|
|
360
|
+
const plannedStep = plannedStepFor(step);
|
|
361
|
+
if (cancelBeforeStepStart(step.id, stepIndex))
|
|
543
362
|
break;
|
|
544
|
-
|
|
363
|
+
const graph = compiledStepGraph(compiled, step);
|
|
545
364
|
const disposition = decideStepDisposition({
|
|
546
365
|
dryRun,
|
|
547
366
|
graph,
|
|
548
367
|
planned: plannedStep,
|
|
549
|
-
reportsByStepId,
|
|
368
|
+
reportsByStepId: state.reportsByStepId,
|
|
550
369
|
step,
|
|
551
370
|
});
|
|
552
371
|
if (disposition.kind === "skip") {
|
|
553
|
-
|
|
554
|
-
publishStepReport(plannedStep, report);
|
|
372
|
+
state.skipStep(plannedStep, disposition);
|
|
555
373
|
continue;
|
|
556
374
|
}
|
|
557
375
|
const inputEntries = [];
|
|
558
|
-
for (const dep of graph.dependsOn)
|
|
559
|
-
inputEntries.push([dep.id, outputs.get(dep.id)]);
|
|
560
|
-
}
|
|
376
|
+
for (const dep of graph.dependsOn)
|
|
377
|
+
inputEntries.push([dep.id, state.outputs.get(dep.id)]);
|
|
561
378
|
for (const dep of graph.optionalDependsOn) {
|
|
562
|
-
if (outputs.has(dep.id))
|
|
563
|
-
inputEntries.push([dep.id, outputs.get(dep.id)]);
|
|
564
|
-
}
|
|
379
|
+
if (state.outputs.has(dep.id))
|
|
380
|
+
inputEntries.push([dep.id, state.outputs.get(dep.id)]);
|
|
565
381
|
}
|
|
566
|
-
const
|
|
382
|
+
const stepInputs = Object.fromEntries(inputEntries);
|
|
567
383
|
if (typeof step.skip === "function") {
|
|
568
384
|
let skipDecision;
|
|
569
385
|
try {
|
|
570
|
-
skipDecision =
|
|
386
|
+
skipDecision = await evaluateStepSkip(step, stepInputs, {
|
|
387
|
+
...executionContext,
|
|
388
|
+
log: tracedLogger(step.id),
|
|
389
|
+
});
|
|
571
390
|
}
|
|
572
391
|
catch (error) {
|
|
573
|
-
const
|
|
574
|
-
|
|
575
|
-
attemptId: skipAttempt.attemptId,
|
|
576
|
-
pipelineId: compiled.id,
|
|
577
|
-
status: "running",
|
|
578
|
-
step: plannedStep,
|
|
579
|
-
});
|
|
580
|
-
if (recordStepExecutionFailure(error, plannedStep, step, stepId, stepIndex, skipAttempt)) {
|
|
392
|
+
const attempt = state.beginAttempt(plannedStep);
|
|
393
|
+
if (recordStepExecutionFailure(error, plannedStep, stepIndex, attempt))
|
|
581
394
|
break;
|
|
582
|
-
}
|
|
583
395
|
continue;
|
|
584
396
|
}
|
|
585
|
-
if (cancelBeforeStepStart(
|
|
397
|
+
if (cancelBeforeStepStart(step.id, stepIndex))
|
|
586
398
|
break;
|
|
587
|
-
}
|
|
588
399
|
if (skipDecision) {
|
|
589
|
-
if (await
|
|
590
|
-
kind: "policy-skip",
|
|
591
|
-
plannedStep,
|
|
592
|
-
skipMessage: skipDecision.reason,
|
|
593
|
-
step,
|
|
594
|
-
stepId,
|
|
595
|
-
stepIndex,
|
|
596
|
-
value: skipDecision.value,
|
|
597
|
-
})) {
|
|
400
|
+
if (await recordPolicySkip(plannedStep, step, stepIndex, skipDecision.reason, skipDecision.value)) {
|
|
598
401
|
break;
|
|
599
402
|
}
|
|
600
403
|
continue;
|
|
601
404
|
}
|
|
602
405
|
}
|
|
603
|
-
const
|
|
604
|
-
const attempt = beginAttempt(stepId, stepStartedAt);
|
|
605
|
-
publishStepStatus({
|
|
606
|
-
attemptId: attempt.attemptId,
|
|
607
|
-
pipelineId: compiled.id,
|
|
608
|
-
status: "running",
|
|
609
|
-
step: plannedStep,
|
|
610
|
-
});
|
|
406
|
+
const attempt = state.beginAttempt(plannedStep);
|
|
611
407
|
try {
|
|
612
|
-
|
|
613
|
-
const publishProgress = (progress) => {
|
|
614
|
-
if (!acceptsProgress)
|
|
615
|
-
return;
|
|
616
|
-
publishStepStatus({
|
|
617
|
-
attemptId: attempt.attemptId,
|
|
618
|
-
pipelineId: compiled.id,
|
|
619
|
-
progress,
|
|
620
|
-
status: "running",
|
|
621
|
-
step: plannedStep,
|
|
622
|
-
});
|
|
623
|
-
};
|
|
624
|
-
const stepContext = {
|
|
625
|
-
...executionContext,
|
|
408
|
+
const output = await executeStepAttempt({
|
|
626
409
|
attemptId: attempt.attemptId,
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
}
|
|
634
|
-
};
|
|
635
|
-
let output;
|
|
636
|
-
try {
|
|
637
|
-
output =
|
|
638
|
-
dryRun && typeof step.dryRun === "function"
|
|
639
|
-
? await step.dryRun(inputs, stepContext)
|
|
640
|
-
: await step.run(inputs, stepContext);
|
|
641
|
-
}
|
|
642
|
-
finally {
|
|
643
|
-
acceptsProgress = false;
|
|
644
|
-
}
|
|
645
|
-
if (await recordPublishedStepValue({
|
|
646
|
-
attempt,
|
|
647
|
-
kind: "complete",
|
|
648
|
-
plannedStep,
|
|
410
|
+
context: executionContext,
|
|
411
|
+
dryRun,
|
|
412
|
+
inputs: stepInputs,
|
|
413
|
+
log: tracedLogger(step.id, attempt.attemptId),
|
|
414
|
+
onProgress: (progress) => state.reportProgress(plannedStep, attempt, progress),
|
|
415
|
+
onReportAttempt: (number, attributes) => lifecycle.reportAttempt(step.id, number, attributes, attempt.attemptId),
|
|
416
|
+
outputBoundary: `Pipeline ${compiled.id} step ${step.id} output`,
|
|
649
417
|
step,
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
value: output,
|
|
653
|
-
})) {
|
|
654
|
-
break;
|
|
655
|
-
}
|
|
418
|
+
});
|
|
419
|
+
state.completeStep(plannedStep, attempt, output);
|
|
656
420
|
}
|
|
657
421
|
catch (error) {
|
|
658
|
-
if (recordStepExecutionFailure(error, plannedStep,
|
|
422
|
+
if (recordStepExecutionFailure(error, plannedStep, stepIndex, attempt))
|
|
659
423
|
break;
|
|
660
|
-
}
|
|
661
424
|
}
|
|
662
425
|
}
|
|
663
|
-
|
|
664
|
-
let finalized = false;
|
|
665
|
-
if (completed || controls.continueOnError) {
|
|
426
|
+
if (state.errors.length === 0 || controls.continueOnError) {
|
|
666
427
|
const finalizeStartedAt = runtime.now();
|
|
667
|
-
|
|
428
|
+
state.beginFinalization();
|
|
668
429
|
try {
|
|
669
430
|
throwIfAborted(runtime);
|
|
670
|
-
// SAFETY: `outputs` maps step ids to their produced values, which is
|
|
431
|
+
// SAFETY: `state.outputs` maps step ids to their produced values, which is
|
|
671
432
|
// exactly the shape the finalizer's outputs parameter describes.
|
|
672
|
-
const finalOutputs = Object.fromEntries(outputs);
|
|
433
|
+
const finalOutputs = Object.fromEntries(state.outputs);
|
|
673
434
|
const finalizedValue = await compiled.finalize(finalOutputs, {
|
|
674
435
|
...executionContext,
|
|
675
436
|
log: tracedLogger(PIPELINE_FINALIZE_STEP_ID),
|
|
@@ -677,11 +438,10 @@ export async function executePlannedRun(input) {
|
|
|
677
438
|
// SAFETY: with a result schema the value is validated against
|
|
678
439
|
// `TPipelineResult`; without one the finalizer's declared return type
|
|
679
440
|
// is `TPipelineResult`, so the cast only restores that type.
|
|
680
|
-
value = compiled.resultSchema
|
|
441
|
+
const value = compiled.resultSchema
|
|
681
442
|
? (await validateStandardSchema(compiled.resultSchema, finalizedValue, `Pipeline ${compiled.id} final result`))
|
|
682
443
|
: finalizedValue;
|
|
683
|
-
|
|
684
|
-
lifecycle.finalizeComplete(runtime.now() - finalizeStartedAt, value);
|
|
444
|
+
state.completeFinalization(value, runtime.now() - finalizeStartedAt);
|
|
685
445
|
}
|
|
686
446
|
catch (error) {
|
|
687
447
|
const cancelled = isPipelineCancellation(error, runtime);
|
|
@@ -696,30 +456,10 @@ export async function executePlannedRun(input) {
|
|
|
696
456
|
phase: "finalization",
|
|
697
457
|
stepId: PIPELINE_FINALIZE_STEP_ID,
|
|
698
458
|
});
|
|
699
|
-
|
|
700
|
-
lifecycle.finalizeError(pipelineError, runtime.now() - finalizeStartedAt);
|
|
701
|
-
completed = false;
|
|
459
|
+
state.failFinalization(pipelineError, runtime.now() - finalizeStartedAt);
|
|
702
460
|
}
|
|
703
461
|
}
|
|
704
|
-
|
|
705
|
-
const result = {
|
|
706
|
-
pipelineId: compiled.id,
|
|
707
|
-
dryRun,
|
|
708
|
-
errors,
|
|
709
|
-
finalized,
|
|
710
|
-
finishedAtMs,
|
|
711
|
-
runId,
|
|
712
|
-
startedAtMs: startedAt,
|
|
713
|
-
status: terminalRunStatus(errors),
|
|
714
|
-
steps: reports,
|
|
715
|
-
value,
|
|
716
|
-
version: RUN_MODEL_VERSION,
|
|
717
|
-
};
|
|
718
|
-
if (parentRunId)
|
|
719
|
-
result.parentRunId = parentRunId;
|
|
720
|
-
lifecycle.pipelineComplete(result);
|
|
721
|
-
await lifecycle.flush();
|
|
722
|
-
return result;
|
|
462
|
+
return state.finish();
|
|
723
463
|
}
|
|
724
464
|
const fallbackNow = () => Date.now();
|
|
725
465
|
const fallbackSleep = (durationMs, signal) => abortableSleep(durationMs, signal, "Pipeline sleep");
|