stageflow 0.31.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/agent/fakeAgent.js +0 -6
- package/dist/agent/piAdapter.js +21 -0
- package/dist/agent/port.d.ts +2 -0
- package/dist/agent/providerAuth.d.ts +6 -0
- package/dist/agent/providerAuth.js +50 -14
- package/dist/agent/resolveAgentPort.d.ts +2 -0
- package/dist/agent/resolveAgentPort.js +5 -1
- package/dist/config/listConfig.d.ts +0 -16
- package/dist/config/listConfig.js +0 -3
- package/dist/config/loadStage.js +10 -0
- package/dist/config/pipelineStageKeys.js +1 -0
- package/dist/email/accounts.d.ts +109 -0
- package/dist/email/accounts.js +285 -0
- package/dist/email/adapter.d.ts +69 -0
- package/dist/email/adapter.js +720 -0
- package/dist/email/attachments.d.ts +21 -0
- package/dist/email/attachments.js +146 -0
- package/dist/email/events.d.ts +95 -0
- package/dist/email/events.js +292 -0
- package/dist/email/host.d.ts +15 -0
- package/dist/email/host.js +119 -0
- package/dist/email/messages.d.ts +44 -0
- package/dist/email/messages.js +182 -0
- package/dist/email/port.d.ts +161 -0
- package/dist/email/port.js +12 -0
- package/dist/email/replies.d.ts +5 -0
- package/dist/email/replies.js +56 -0
- package/dist/email/submissions.d.ts +31 -0
- package/dist/email/submissions.js +123 -0
- package/dist/email/triggers.d.ts +140 -0
- package/dist/email/triggers.js +563 -0
- package/dist/hosted/containment.d.ts +3 -0
- package/dist/hosted/containment.js +36 -0
- package/dist/hosted/errors.d.ts +21 -0
- package/dist/hosted/errors.js +30 -0
- package/dist/hosted/packageSkills.d.ts +3 -0
- package/dist/hosted/packageSkills.js +80 -0
- package/dist/hosted/runtime.d.ts +12 -0
- package/dist/hosted/runtime.js +596 -0
- package/dist/hosted/sequencer.d.ts +28 -0
- package/dist/hosted/sequencer.js +129 -0
- package/dist/hosted/translate.d.ts +3 -0
- package/dist/hosted/translate.js +43 -0
- package/dist/hosted/types.d.ts +108 -0
- package/dist/hosted/types.js +10 -0
- package/dist/index.d.ts +8 -1
- package/dist/index.js +5 -0
- package/dist/package-meta.d.ts +1 -1
- package/dist/package-meta.js +1 -1
- package/dist/projection/projectRun.d.ts +0 -1
- package/dist/projection/projectRun.js +0 -1
- package/dist/runstore/port.d.ts +4 -0
- package/dist/runstore/sqlite/SqliteRunStore.d.ts +1 -0
- package/dist/runstore/sqlite/SqliteRunStore.js +24 -5
- package/dist/runstore/sqlite/migrations/011-email-dispatch-key.d.ts +7 -0
- package/dist/runstore/sqlite/migrations/011-email-dispatch-key.js +12 -0
- package/dist/runstore/sqlite/migrations/index.js +2 -0
- package/dist/runtime/containerLimits.d.ts +0 -3
- package/dist/runtime/containerLimits.js +0 -5
- package/dist/runtime/credentialBinding.d.ts +3 -0
- package/dist/runtime/credentialBinding.js +9 -0
- package/dist/runtime/pipelineRunner.d.ts +3 -0
- package/dist/runtime/pipelineRunner.js +11 -0
- package/dist/runtime/pipelineScheduler.js +3 -0
- package/dist/runtime/runManager.d.ts +1 -0
- package/dist/runtime/runManager.js +24 -3
- package/dist/runtime/runRetentionSweep.d.ts +0 -2
- package/dist/runtime/runRetentionSweep.js +0 -8
- package/dist/runtime/stageAttemptBootstrap.js +14 -0
- package/dist/runtime/stageProcessLauncher.d.ts +4 -0
- package/dist/runtime/stageProcessLauncher.js +45 -1
- package/dist/runtime/stageRoots.js +5 -1
- package/dist/server/bootstrap.d.ts +7 -0
- package/dist/server/bootstrap.js +17 -0
- package/dist/server/emailRoutes.d.ts +5 -0
- package/dist/server/emailRoutes.js +136 -0
- package/dist/server/http.d.ts +2 -0
- package/dist/server/http.js +16 -0
- package/dist/tools/readEmail.d.ts +77 -0
- package/dist/tools/readEmail.js +31 -0
- package/dist/tools/replyEmail.d.ts +31 -0
- package/dist/tools/replyEmail.js +25 -0
- package/dist/tools/sendEmail.d.ts +43 -0
- package/dist/tools/sendEmail.js +26 -0
- package/dist/types/stage.d.ts +4 -0
- package/dist/ui/assets/index-59cjF0OA.js +133 -0
- package/dist/ui/index.html +1 -1
- package/examples/hello-email/README.md +23 -0
- package/examples/hello-email/hello-email-reply.pipeline.yaml +6 -0
- package/examples/hello-email/hello-email.pipeline.yaml +6 -0
- package/examples/hello-email/hello-email.task.yaml +6 -0
- package/examples/hello-email/reply-hello-email.yaml +63 -0
- package/examples/hello-email/send-hello-email.yaml +37 -0
- package/package.json +14 -2
- package/dist/ui/assets/index-p8Iou3zk.js +0 -133
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type HostedEventBody, type HostedEventSink, type HostedRunContext } from "./types.js";
|
|
2
|
+
export declare class HostedEventSequencer {
|
|
3
|
+
private readonly sink;
|
|
4
|
+
private readonly context;
|
|
5
|
+
private readonly timeoutMs;
|
|
6
|
+
private readonly attempts;
|
|
7
|
+
private chain;
|
|
8
|
+
private held;
|
|
9
|
+
private holding;
|
|
10
|
+
private sequence;
|
|
11
|
+
private runId;
|
|
12
|
+
failed: boolean;
|
|
13
|
+
failure?: {
|
|
14
|
+
eventType: string;
|
|
15
|
+
sequence: number;
|
|
16
|
+
message: string;
|
|
17
|
+
};
|
|
18
|
+
constructor(sink: HostedEventSink, context: HostedRunContext, timeoutMs: number, attempts: number);
|
|
19
|
+
setRunId(runId: string): void;
|
|
20
|
+
emit(body: HostedEventBody): Promise<void>;
|
|
21
|
+
emitNow(body: HostedEventBody): Promise<void>;
|
|
22
|
+
release(): void;
|
|
23
|
+
rejectHeld(error: unknown): void;
|
|
24
|
+
private enqueue;
|
|
25
|
+
private deliver;
|
|
26
|
+
private envelope;
|
|
27
|
+
private failureError;
|
|
28
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { redactString } from "../logging/redact.js";
|
|
2
|
+
import { HostedRuntimeError } from "./errors.js";
|
|
3
|
+
import { HOSTED_EVENT_SCHEMA_VERSION, } from "./types.js";
|
|
4
|
+
export class HostedEventSequencer {
|
|
5
|
+
sink;
|
|
6
|
+
context;
|
|
7
|
+
timeoutMs;
|
|
8
|
+
attempts;
|
|
9
|
+
chain = Promise.resolve();
|
|
10
|
+
held = [];
|
|
11
|
+
holding = true;
|
|
12
|
+
sequence = 0;
|
|
13
|
+
runId = "";
|
|
14
|
+
failed = false;
|
|
15
|
+
failure;
|
|
16
|
+
constructor(sink, context, timeoutMs, attempts) {
|
|
17
|
+
this.sink = sink;
|
|
18
|
+
this.context = context;
|
|
19
|
+
this.timeoutMs = timeoutMs;
|
|
20
|
+
this.attempts = attempts;
|
|
21
|
+
}
|
|
22
|
+
setRunId(runId) {
|
|
23
|
+
this.runId = runId;
|
|
24
|
+
}
|
|
25
|
+
emit(body) {
|
|
26
|
+
if (this.failed)
|
|
27
|
+
return Promise.reject(this.failureError());
|
|
28
|
+
if (this.holding) {
|
|
29
|
+
return new Promise((resolve, reject) => {
|
|
30
|
+
this.held.push({ body, resolve, reject });
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return this.enqueue(body);
|
|
34
|
+
}
|
|
35
|
+
emitNow(body) {
|
|
36
|
+
if (this.failed)
|
|
37
|
+
return Promise.reject(this.failureError());
|
|
38
|
+
return this.enqueue(body);
|
|
39
|
+
}
|
|
40
|
+
release() {
|
|
41
|
+
if (!this.holding)
|
|
42
|
+
return;
|
|
43
|
+
this.holding = false;
|
|
44
|
+
const queued = this.held;
|
|
45
|
+
this.held = [];
|
|
46
|
+
for (const item of queued) {
|
|
47
|
+
const run = this.chain.then(() => this.deliver(item.body));
|
|
48
|
+
this.chain = run.then(() => undefined, () => undefined);
|
|
49
|
+
void run.then(item.resolve, item.reject);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
rejectHeld(error) {
|
|
53
|
+
this.holding = false;
|
|
54
|
+
const queued = this.held;
|
|
55
|
+
this.held = [];
|
|
56
|
+
for (const item of queued)
|
|
57
|
+
item.reject(error);
|
|
58
|
+
}
|
|
59
|
+
enqueue(body) {
|
|
60
|
+
const run = this.chain.then(() => this.deliver(body));
|
|
61
|
+
this.chain = run.then(() => undefined, () => undefined);
|
|
62
|
+
return run;
|
|
63
|
+
}
|
|
64
|
+
async deliver(body) {
|
|
65
|
+
if (this.failed)
|
|
66
|
+
throw this.failureError();
|
|
67
|
+
const sequence = ++this.sequence;
|
|
68
|
+
const event = this.envelope(body, sequence);
|
|
69
|
+
let last;
|
|
70
|
+
for (let attempt = 1; attempt <= this.attempts; attempt += 1) {
|
|
71
|
+
try {
|
|
72
|
+
await withTimeout(this.sink.onEvent(event), this.timeoutMs);
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
last = error;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const message = redactString(errorMessage(last));
|
|
80
|
+
this.failed = true;
|
|
81
|
+
this.failure = { eventType: event.type, sequence, message };
|
|
82
|
+
throw this.failureError();
|
|
83
|
+
}
|
|
84
|
+
envelope(body, sequence) {
|
|
85
|
+
return {
|
|
86
|
+
schemaVersion: HOSTED_EVENT_SCHEMA_VERSION,
|
|
87
|
+
eventId: `${this.context.externalAttemptId}:${sequence}`,
|
|
88
|
+
sequence,
|
|
89
|
+
occurredAt: new Date().toISOString(),
|
|
90
|
+
runId: this.runId,
|
|
91
|
+
externalRunId: this.context.externalRunId,
|
|
92
|
+
externalAttemptId: this.context.externalAttemptId,
|
|
93
|
+
packageRevision: this.context.packageRevision,
|
|
94
|
+
...(this.context.attributes !== undefined
|
|
95
|
+
? { attributes: this.context.attributes }
|
|
96
|
+
: {}),
|
|
97
|
+
...body,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
failureError() {
|
|
101
|
+
const failure = this.failure;
|
|
102
|
+
return new HostedRuntimeError(failure?.message ?? "hosted event sink failed", {
|
|
103
|
+
code: "event_sink_failed",
|
|
104
|
+
externalRunId: this.context.externalRunId,
|
|
105
|
+
externalAttemptId: this.context.externalAttemptId,
|
|
106
|
+
...(this.runId !== "" ? { runId: this.runId } : {}),
|
|
107
|
+
...(failure !== undefined
|
|
108
|
+
? { eventType: failure.eventType, sequence: failure.sequence }
|
|
109
|
+
: {}),
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
function errorMessage(error) {
|
|
114
|
+
return error instanceof Error ? error.message : String(error);
|
|
115
|
+
}
|
|
116
|
+
function withTimeout(work, timeoutMs) {
|
|
117
|
+
return new Promise((resolve, reject) => {
|
|
118
|
+
const timer = setTimeout(() => {
|
|
119
|
+
reject(new Error("hosted event sink timed out"));
|
|
120
|
+
}, timeoutMs);
|
|
121
|
+
work.then(() => {
|
|
122
|
+
clearTimeout(timer);
|
|
123
|
+
resolve();
|
|
124
|
+
}, (error) => {
|
|
125
|
+
clearTimeout(timer);
|
|
126
|
+
reject(error);
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { redactString } from "../logging/redact.js";
|
|
2
|
+
const ACTIVITY_EVENTS = new Set([
|
|
3
|
+
"agent_start",
|
|
4
|
+
"agent_end",
|
|
5
|
+
"turn_start",
|
|
6
|
+
"tool_start",
|
|
7
|
+
"tool_end",
|
|
8
|
+
"tool_progress",
|
|
9
|
+
"message",
|
|
10
|
+
"operator_prompt",
|
|
11
|
+
"operator_answer",
|
|
12
|
+
]);
|
|
13
|
+
function redactActivity(event) {
|
|
14
|
+
return JSON.parse(redactString(JSON.stringify(event)));
|
|
15
|
+
}
|
|
16
|
+
export function translateStageEvent(stageId, event, attempt) {
|
|
17
|
+
if (ACTIVITY_EVENTS.has(event.event)) {
|
|
18
|
+
return {
|
|
19
|
+
type: "stage.activity",
|
|
20
|
+
stageId,
|
|
21
|
+
attempt,
|
|
22
|
+
activity: redactActivity(event),
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
switch (event.event) {
|
|
26
|
+
case "started":
|
|
27
|
+
return { type: "stage.started", stageId, attempt };
|
|
28
|
+
case "waiting_for_input":
|
|
29
|
+
return { type: "stage.waiting", stageId, attempt };
|
|
30
|
+
case "succeeded":
|
|
31
|
+
return { type: "stage.succeeded", stageId, attempt };
|
|
32
|
+
case "failed":
|
|
33
|
+
case "interrupted":
|
|
34
|
+
return {
|
|
35
|
+
type: "stage.failed",
|
|
36
|
+
stageId,
|
|
37
|
+
attempt,
|
|
38
|
+
reason: redactString(event.reason),
|
|
39
|
+
};
|
|
40
|
+
default:
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type { StageActivityEvent } from "../agent/activity.js";
|
|
2
|
+
import type { HostedErrorCode } from "./errors.js";
|
|
3
|
+
export declare const HOSTED_EVENT_SCHEMA_VERSION: 1;
|
|
4
|
+
/** One sink call is attempted this many times. Delivery is at least once, not exactly once. */
|
|
5
|
+
export declare const HOSTED_DELIVERY_ATTEMPTS = 2;
|
|
6
|
+
/** Bound for a single sink call. The private sink and Cloud worker own further retries. */
|
|
7
|
+
export declare const DEFAULT_EVENT_DELIVERY_TIMEOUT_MS = 10000;
|
|
8
|
+
/**
|
|
9
|
+
* After cancellation or timeout, Core waits at most this long for in-flight
|
|
10
|
+
* stage work to stop. `AgentPort` handles must stop inside `close()`.
|
|
11
|
+
*/
|
|
12
|
+
export declare const HOSTED_CANCEL_GRACE_MS = 5000;
|
|
13
|
+
export type HostedRunContext = {
|
|
14
|
+
/** Opaque Cloud run id. Core copies it and does not interpret it. */
|
|
15
|
+
externalRunId: string;
|
|
16
|
+
/** Opaque Cloud attempt id. Core copies it and does not interpret it. */
|
|
17
|
+
externalAttemptId: string;
|
|
18
|
+
/** Revision of the package Cloud already materialized. */
|
|
19
|
+
packageRevision: string;
|
|
20
|
+
/** Extra non-secret correlation strings. */
|
|
21
|
+
attributes?: Readonly<Record<string, string>>;
|
|
22
|
+
};
|
|
23
|
+
export type HostedPipelineInput = {
|
|
24
|
+
packageRoot: string;
|
|
25
|
+
pipeline: string;
|
|
26
|
+
taskYaml: string;
|
|
27
|
+
context: HostedRunContext;
|
|
28
|
+
signal?: AbortSignal;
|
|
29
|
+
maxActiveStagesPerRun?: number;
|
|
30
|
+
/** Wall-clock limit for the hosted run. Omit for no hosted-level limit. */
|
|
31
|
+
timeoutMs?: number;
|
|
32
|
+
};
|
|
33
|
+
export type HostedExecutionReceipt = {
|
|
34
|
+
coreVersion: string;
|
|
35
|
+
eventSchemaVersion: typeof HOSTED_EVENT_SCHEMA_VERSION;
|
|
36
|
+
packageRevision: string;
|
|
37
|
+
executionMode: "inline";
|
|
38
|
+
startedAt: string;
|
|
39
|
+
finishedAt: string;
|
|
40
|
+
};
|
|
41
|
+
export type HostedPipelineResult = {
|
|
42
|
+
/** Null when the run failed before Core created a local run. */
|
|
43
|
+
runId: string | null;
|
|
44
|
+
externalRunId: string;
|
|
45
|
+
externalAttemptId: string;
|
|
46
|
+
status: "succeeded" | "failed" | "cancelled" | "waiting";
|
|
47
|
+
code?: HostedErrorCode;
|
|
48
|
+
reason?: string;
|
|
49
|
+
/** Present once a local run exists. Caller exports artifacts from here. */
|
|
50
|
+
workspaceDir?: string;
|
|
51
|
+
receipt: HostedExecutionReceipt;
|
|
52
|
+
};
|
|
53
|
+
export type HostedEventEnvelope = {
|
|
54
|
+
schemaVersion: typeof HOSTED_EVENT_SCHEMA_VERSION;
|
|
55
|
+
eventId: string;
|
|
56
|
+
sequence: number;
|
|
57
|
+
occurredAt: string;
|
|
58
|
+
runId: string;
|
|
59
|
+
externalRunId: string;
|
|
60
|
+
externalAttemptId: string;
|
|
61
|
+
packageRevision: string;
|
|
62
|
+
attributes?: Readonly<Record<string, string>>;
|
|
63
|
+
};
|
|
64
|
+
export type HostedRunEvent = HostedEventEnvelope & ({
|
|
65
|
+
type: "run.created";
|
|
66
|
+
} | {
|
|
67
|
+
type: "run.started";
|
|
68
|
+
} | {
|
|
69
|
+
type: "stage.started";
|
|
70
|
+
stageId: string;
|
|
71
|
+
attempt: number;
|
|
72
|
+
} | {
|
|
73
|
+
type: "stage.activity";
|
|
74
|
+
stageId: string;
|
|
75
|
+
attempt: number;
|
|
76
|
+
activity: StageActivityEvent;
|
|
77
|
+
} | {
|
|
78
|
+
type: "stage.waiting";
|
|
79
|
+
stageId: string;
|
|
80
|
+
attempt: number;
|
|
81
|
+
} | {
|
|
82
|
+
type: "stage.succeeded";
|
|
83
|
+
stageId: string;
|
|
84
|
+
attempt: number;
|
|
85
|
+
} | {
|
|
86
|
+
type: "stage.failed";
|
|
87
|
+
stageId: string;
|
|
88
|
+
attempt: number;
|
|
89
|
+
reason?: string;
|
|
90
|
+
} | {
|
|
91
|
+
type: "run.succeeded";
|
|
92
|
+
receipt: HostedExecutionReceipt;
|
|
93
|
+
} | {
|
|
94
|
+
type: "run.failed";
|
|
95
|
+
code?: HostedErrorCode;
|
|
96
|
+
reason?: string;
|
|
97
|
+
receipt: HostedExecutionReceipt;
|
|
98
|
+
} | {
|
|
99
|
+
type: "run.cancelled";
|
|
100
|
+
reason?: string;
|
|
101
|
+
receipt: HostedExecutionReceipt;
|
|
102
|
+
});
|
|
103
|
+
type DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K> : never;
|
|
104
|
+
export type HostedEventBody = DistributiveOmit<HostedRunEvent, keyof HostedEventEnvelope>;
|
|
105
|
+
export interface HostedEventSink {
|
|
106
|
+
onEvent(event: HostedRunEvent): Promise<void>;
|
|
107
|
+
}
|
|
108
|
+
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export const HOSTED_EVENT_SCHEMA_VERSION = 1;
|
|
2
|
+
/** One sink call is attempted this many times. Delivery is at least once, not exactly once. */
|
|
3
|
+
export const HOSTED_DELIVERY_ATTEMPTS = 2;
|
|
4
|
+
/** Bound for a single sink call. The private sink and Cloud worker own further retries. */
|
|
5
|
+
export const DEFAULT_EVENT_DELIVERY_TIMEOUT_MS = 10_000;
|
|
6
|
+
/**
|
|
7
|
+
* After cancellation or timeout, Core waits at most this long for in-flight
|
|
8
|
+
* stage work to stop. `AgentPort` handles must stop inside `close()`.
|
|
9
|
+
*/
|
|
10
|
+
export const HOSTED_CANCEL_GRACE_MS = 5_000;
|
package/dist/index.d.ts
CHANGED
|
@@ -5,10 +5,17 @@ export { resolveStageflowContext, projectContextFromStageflow, type StageflowCon
|
|
|
5
5
|
export { PACKAGE_NAME, PACKAGE_VERSION } from "./package-meta.js";
|
|
6
6
|
export type { StageEnvelope } from "./types/envelope.js";
|
|
7
7
|
export { assertRequiredEnvelope, isAdvancingEnvelope, } from "./envelope/check.js";
|
|
8
|
-
export type { AgentPort } from "./agent/port.js";
|
|
8
|
+
export type { AgentPort, StageHandle, StageHandleEvent, StageRunInput, StageRunResult, } from "./agent/port.js";
|
|
9
|
+
export { createPiAgentPort } from "./agent/resolveAgentPort.js";
|
|
10
|
+
export { configurePiProviderApiKey } from "./agent/providerAuth.js";
|
|
9
11
|
export type { StageActivityEvent, StageLogLine } from "./agent/activity.js";
|
|
10
12
|
export type { StageRoots } from "./runtime/stageRoots.js";
|
|
11
13
|
export { runPipeline, startPipeline } from "./runtime/pipelineRunner.js";
|
|
14
|
+
export { createHostedRuntime, type HostedRuntime, } from "./hosted/runtime.js";
|
|
15
|
+
export { HostedRuntimeError, HOSTED_ERROR_CODES } from "./hosted/errors.js";
|
|
16
|
+
export type { HostedErrorCode } from "./hosted/errors.js";
|
|
17
|
+
export { HOSTED_EVENT_SCHEMA_VERSION, HOSTED_DELIVERY_ATTEMPTS, DEFAULT_EVENT_DELIVERY_TIMEOUT_MS, HOSTED_CANCEL_GRACE_MS, } from "./hosted/types.js";
|
|
18
|
+
export type { HostedRunContext, HostedPipelineInput, HostedPipelineResult, HostedExecutionReceipt, HostedEventEnvelope, HostedRunEvent, HostedEventSink, } from "./hosted/types.js";
|
|
12
19
|
export { createRunStore } from "./runstore/createStore.js";
|
|
13
20
|
export type { RunStoreConfig, RunStoreKind } from "./runstore/createStore.js";
|
|
14
21
|
export type { RunStore, RunSummary, RunDetail, RunMeta, RunStatus, StageSnapshot, StageLogEvent, CreatedRun, } from "./runstore/port.js";
|
package/dist/index.js
CHANGED
|
@@ -4,7 +4,12 @@ export { resolveProjectContext, legacyProjectContext, } from "./project/resolveP
|
|
|
4
4
|
export { resolveStageflowContext, projectContextFromStageflow, } from "./project/resolveStageflowContext.js";
|
|
5
5
|
export { PACKAGE_NAME, PACKAGE_VERSION } from "./package-meta.js";
|
|
6
6
|
export { assertRequiredEnvelope, isAdvancingEnvelope, } from "./envelope/check.js";
|
|
7
|
+
export { createPiAgentPort } from "./agent/resolveAgentPort.js";
|
|
8
|
+
export { configurePiProviderApiKey } from "./agent/providerAuth.js";
|
|
7
9
|
export { runPipeline, startPipeline } from "./runtime/pipelineRunner.js";
|
|
10
|
+
export { createHostedRuntime, } from "./hosted/runtime.js";
|
|
11
|
+
export { HostedRuntimeError, HOSTED_ERROR_CODES } from "./hosted/errors.js";
|
|
12
|
+
export { HOSTED_EVENT_SCHEMA_VERSION, HOSTED_DELIVERY_ATTEMPTS, DEFAULT_EVENT_DELIVERY_TIMEOUT_MS, HOSTED_CANCEL_GRACE_MS, } from "./hosted/types.js";
|
|
8
13
|
export { createRunStore } from "./runstore/createStore.js";
|
|
9
14
|
export { deriveStatusFromStages } from "./runstore/port.js";
|
|
10
15
|
export { RunManager } from "./runtime/runManager.js";
|
package/dist/package-meta.d.ts
CHANGED
package/dist/package-meta.js
CHANGED
package/dist/runstore/port.d.ts
CHANGED
|
@@ -78,6 +78,8 @@ export type RunMeta = {
|
|
|
78
78
|
};
|
|
79
79
|
export type CreatedRun = {
|
|
80
80
|
runId: string;
|
|
81
|
+
/** False when a durable dispatch key already belongs to this run. */
|
|
82
|
+
created?: boolean;
|
|
81
83
|
/** Opaque agent workspace path (artifacts live under this tree). */
|
|
82
84
|
workspaceDir: string;
|
|
83
85
|
};
|
|
@@ -338,6 +340,7 @@ export type FeedbackLoopHistory = {
|
|
|
338
340
|
fork_generations: ForkGenerationRecord[];
|
|
339
341
|
};
|
|
340
342
|
export type CreateRunInput = {
|
|
343
|
+
dispatchKey?: string;
|
|
341
344
|
submission?: RunSubmission;
|
|
342
345
|
runId?: string;
|
|
343
346
|
pipelineId: string;
|
|
@@ -396,6 +399,7 @@ export interface RunStore {
|
|
|
396
399
|
}>;
|
|
397
400
|
getRunBySubmission(key: string): Promise<RunSubmissionRecord | null>;
|
|
398
401
|
createRun(input: CreateRunInput): Promise<CreatedRun>;
|
|
402
|
+
findRunByDispatchKey?(dispatchKey: string): Promise<CreatedRun | undefined>;
|
|
399
403
|
updateRunStatus(runId: string, status: RunStatus): Promise<void>;
|
|
400
404
|
/**
|
|
401
405
|
* Conditional status update (CAS): sets `status` only when the row currently
|
|
@@ -24,6 +24,7 @@ export declare class SqliteRunStore implements RunStore {
|
|
|
24
24
|
get connection(): Database.Database;
|
|
25
25
|
getWorkspaceDir(runId: string): string;
|
|
26
26
|
createRun(input: CreateRunInput): Promise<CreatedRun>;
|
|
27
|
+
findRunByDispatchKey(dispatchKey: string): Promise<CreatedRun | undefined>;
|
|
27
28
|
private readSubmission;
|
|
28
29
|
getRunBySubmission(key: string): Promise<RunSubmissionRecord | null>;
|
|
29
30
|
updateRunStatus(runId: string, status: RunStatus): Promise<void>;
|
|
@@ -189,9 +189,13 @@ export class SqliteRunStore {
|
|
|
189
189
|
}
|
|
190
190
|
async createRun(input) {
|
|
191
191
|
await this.ready();
|
|
192
|
+
const existing = input.dispatchKey ? await this.findRunByDispatchKey(input.dispatchKey) : undefined;
|
|
193
|
+
if (existing)
|
|
194
|
+
return existing;
|
|
192
195
|
const runId = input.runId ?? newRunId();
|
|
193
196
|
const workspaceDir = this.getWorkspaceDir(runId);
|
|
194
|
-
|
|
197
|
+
if (!input.dispatchKey)
|
|
198
|
+
await mkdir(path.join(workspaceDir, "stages"), { recursive: true });
|
|
195
199
|
const now = new Date().toISOString();
|
|
196
200
|
const pipelinePath = input.pipelinePath
|
|
197
201
|
? normalizeCatalogPath(input.pipelinePath)
|
|
@@ -214,13 +218,15 @@ export class SqliteRunStore {
|
|
|
214
218
|
if (existing)
|
|
215
219
|
throw new RunSubmissionExistsError(existing);
|
|
216
220
|
}
|
|
217
|
-
this.db
|
|
221
|
+
const inserted = this.db
|
|
218
222
|
.prepare(`INSERT INTO runs
|
|
219
|
-
(run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root, repository, ref, resolved_sha, run_branch, git_author_name, git_author_email, pipeline_source, pipeline_body, caller_id, run_manifest, skip_gates)
|
|
223
|
+
(run_id, pipeline_id, task_id, task_yaml, status, created_at, updated_at, checkout_root, pipeline_dag_json, git_sha, ci_pr_url, ci_job_url, pipeline_path, task_path, project_root, repository, ref, resolved_sha, run_branch, git_author_name, git_author_email, pipeline_source, pipeline_body, caller_id, run_manifest, skip_gates, dispatch_key)
|
|
220
224
|
VALUES
|
|
221
|
-
(@run_id, @pipeline_id, @task_id, @task_yaml, @status, @created_at, @updated_at, @checkout_root, @pipeline_dag_json, @git_sha, @ci_pr_url, @ci_job_url, @pipeline_path, @task_path, @project_root, @repository, @ref, @resolved_sha, @run_branch, @git_author_name, @git_author_email, @pipeline_source, @pipeline_body, @caller_id, @run_manifest, @skip_gates)
|
|
225
|
+
(@run_id, @pipeline_id, @task_id, @task_yaml, @status, @created_at, @updated_at, @checkout_root, @pipeline_dag_json, @git_sha, @ci_pr_url, @ci_job_url, @pipeline_path, @task_path, @project_root, @repository, @ref, @resolved_sha, @run_branch, @git_author_name, @git_author_email, @pipeline_source, @pipeline_body, @caller_id, @run_manifest, @skip_gates, @dispatch_key)
|
|
226
|
+
ON CONFLICT(dispatch_key) DO NOTHING`)
|
|
222
227
|
.run({
|
|
223
228
|
run_id: runId,
|
|
229
|
+
dispatch_key: input.dispatchKey ?? null,
|
|
224
230
|
pipeline_id: input.pipelineId,
|
|
225
231
|
task_id: input.taskId ?? null,
|
|
226
232
|
task_yaml: input.taskYaml,
|
|
@@ -251,12 +257,25 @@ export class SqliteRunStore {
|
|
|
251
257
|
: null,
|
|
252
258
|
skip_gates: input.skipGates === undefined ? null : input.skipGates ? 1 : 0,
|
|
253
259
|
});
|
|
260
|
+
if (!inserted.changes)
|
|
261
|
+
return;
|
|
254
262
|
if (input.submission) {
|
|
255
263
|
this.db.prepare("INSERT INTO run_submissions (submission_key, request_hash, run_id) VALUES (?, ?, ?)")
|
|
256
264
|
.run(input.submission.key, input.submission.requestHash, runId);
|
|
257
265
|
}
|
|
258
266
|
})();
|
|
259
|
-
|
|
267
|
+
if (input.dispatchKey) {
|
|
268
|
+
const selected = await this.findRunByDispatchKey(input.dispatchKey);
|
|
269
|
+
if (selected && selected.runId !== runId)
|
|
270
|
+
return selected;
|
|
271
|
+
await mkdir(path.join(workspaceDir, "stages"), { recursive: true });
|
|
272
|
+
}
|
|
273
|
+
return { runId, workspaceDir, ...(input.dispatchKey ? { created: true } : {}) };
|
|
274
|
+
}
|
|
275
|
+
async findRunByDispatchKey(dispatchKey) {
|
|
276
|
+
await this.ready();
|
|
277
|
+
const row = this.db.prepare("SELECT run_id FROM runs WHERE dispatch_key = ?").get(dispatchKey);
|
|
278
|
+
return row ? { runId: row.run_id, workspaceDir: this.getWorkspaceDir(row.run_id), created: false } : undefined;
|
|
260
279
|
}
|
|
261
280
|
readSubmission(key) {
|
|
262
281
|
const row = this.db.prepare("SELECT submission_key AS key, request_hash AS requestHash, run_id AS runId FROM run_submissions WHERE submission_key = ?")
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { PACKAGE_VERSION } from "../../../package-meta.js";
|
|
2
|
+
export const MIGRATION_011 = {
|
|
3
|
+
version: 11,
|
|
4
|
+
name: "011_email_dispatch_key",
|
|
5
|
+
minStageflowVersion: PACKAGE_VERSION,
|
|
6
|
+
up(db) {
|
|
7
|
+
const columns = db.prepare("PRAGMA table_info(runs)").all();
|
|
8
|
+
if (!columns.some(column => column.name === "dispatch_key"))
|
|
9
|
+
db.exec("ALTER TABLE runs ADD COLUMN dispatch_key TEXT");
|
|
10
|
+
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_runs_dispatch_key ON runs(dispatch_key)");
|
|
11
|
+
},
|
|
12
|
+
};
|
|
@@ -9,6 +9,7 @@ import { MIGRATION_007 } from "./007-projects-registry.js";
|
|
|
9
9
|
import { MIGRATION_008 } from "./008-triggers-table.js";
|
|
10
10
|
import { MIGRATION_009 } from "./009-trigger-next-run.js";
|
|
11
11
|
import { MIGRATION_010 } from "./010-trigger-adapter-state.js";
|
|
12
|
+
import { MIGRATION_011 } from "./011-email-dispatch-key.js";
|
|
12
13
|
const DEFAULT_MIGRATIONS = [
|
|
13
14
|
MIGRATION_001,
|
|
14
15
|
MIGRATION_002,
|
|
@@ -20,6 +21,7 @@ const DEFAULT_MIGRATIONS = [
|
|
|
20
21
|
MIGRATION_008,
|
|
21
22
|
MIGRATION_009,
|
|
22
23
|
MIGRATION_010,
|
|
24
|
+
MIGRATION_011,
|
|
23
25
|
];
|
|
24
26
|
export const CURRENT_SCHEMA_VERSION = DEFAULT_MIGRATIONS[DEFAULT_MIGRATIONS.length - 1].version;
|
|
25
27
|
function throwStoreSchemaTooNew(db, version, maxKnown) {
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import os from "node:os";
|
|
2
1
|
export type ContainerLimits = {
|
|
3
2
|
memoryLimitBytes: number | undefined;
|
|
4
3
|
maxActiveStageProcesses: number;
|
|
@@ -12,5 +11,3 @@ export declare function readContainerMemoryLimit(roots?: readonly string[]): {
|
|
|
12
11
|
export declare function deriveContainerLimits(memoryLimitBytes: number | undefined, source?: ContainerLimits["source"]): ContainerLimits;
|
|
13
12
|
export declare function getContainerLimits(): ContainerLimits;
|
|
14
13
|
export declare function resetContainerLimitsForTests(): void;
|
|
15
|
-
/** Test hook: prove sizing never consults os.totalmem. */
|
|
16
|
-
export declare function assertNoOsTotalmemInSizing(): typeof os.totalmem;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { readFileSync, existsSync } from "node:fs";
|
|
2
|
-
import os from "node:os";
|
|
3
2
|
const HOST_RESERVE_MIB = 512;
|
|
4
3
|
const WORKER_BUDGET_MIB = 512;
|
|
5
4
|
const MAX_CAP = 8;
|
|
@@ -78,7 +77,3 @@ export function getContainerLimits() {
|
|
|
78
77
|
export function resetContainerLimitsForTests() {
|
|
79
78
|
cached = undefined;
|
|
80
79
|
}
|
|
81
|
-
/** Test hook: prove sizing never consults os.totalmem. */
|
|
82
|
-
export function assertNoOsTotalmemInSizing() {
|
|
83
|
-
return os.totalmem;
|
|
84
|
-
}
|
|
@@ -19,5 +19,8 @@ export declare function stageflowAgentAuthPath(override?: string): string;
|
|
|
19
19
|
export declare function piHomeAuthPath(override?: string): string;
|
|
20
20
|
export declare function isUsableAuthFile(authPath: string): boolean;
|
|
21
21
|
export declare function ensureSfOwnedAuthStore(): string;
|
|
22
|
+
/** Use this auth file for credential binding and do not create the operator home. */
|
|
23
|
+
export declare function runWithExplicitAuthPath<T>(authPath: string, fn: () => Promise<T>): Promise<T>;
|
|
24
|
+
export declare function currentExplicitAuthPath(): string | undefined;
|
|
22
25
|
export declare function resolveCredentialBinding(ctx: ProjectContext | string, _options?: ResolveCredentialBindingOptions): CredentialBinding;
|
|
23
26
|
export { readCredentialSourceFromFile, writeCredentialSourceToFile, } from "./settingsFile.js";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { ensureGlobalHome } from "../project/globalHome.js";
|
|
4
5
|
import { resolveProjectContext, } from "../project/resolveProjectContext.js";
|
|
@@ -53,6 +54,14 @@ export function ensureSfOwnedAuthStore() {
|
|
|
53
54
|
}
|
|
54
55
|
return authPath;
|
|
55
56
|
}
|
|
57
|
+
const explicitAuthPath = new AsyncLocalStorage();
|
|
58
|
+
/** Use this auth file for credential binding and do not create the operator home. */
|
|
59
|
+
export function runWithExplicitAuthPath(authPath, fn) {
|
|
60
|
+
return explicitAuthPath.run(path.resolve(authPath), fn);
|
|
61
|
+
}
|
|
62
|
+
export function currentExplicitAuthPath() {
|
|
63
|
+
return explicitAuthPath.getStore();
|
|
64
|
+
}
|
|
56
65
|
export function resolveCredentialBinding(ctx, _options = {}) {
|
|
57
66
|
const projectCtx = typeof ctx === "string" ? resolveProjectContext(ctx) : ctx;
|
|
58
67
|
const persisted = readCredentialSourceFromContext(projectCtx);
|
|
@@ -41,6 +41,7 @@ export type PipelineRunResult = {
|
|
|
41
41
|
findings?: ValidationFinding[];
|
|
42
42
|
};
|
|
43
43
|
export type StartedPipeline = {
|
|
44
|
+
created?: boolean;
|
|
44
45
|
runId: string;
|
|
45
46
|
runDir: string;
|
|
46
47
|
done: Promise<PipelineRunResult>;
|
|
@@ -51,6 +52,7 @@ export type PreparedPipeline = {
|
|
|
51
52
|
run: {
|
|
52
53
|
runId: string;
|
|
53
54
|
workspaceDir: string;
|
|
55
|
+
created?: boolean;
|
|
54
56
|
};
|
|
55
57
|
agent: AgentPort;
|
|
56
58
|
store: RunStore;
|
|
@@ -97,6 +99,7 @@ export declare function runPipeline(options: {
|
|
|
97
99
|
}): Promise<PipelineRunResult>;
|
|
98
100
|
/** Create the run immediately, then execute stages in the returned promise. */
|
|
99
101
|
export declare function startPipeline(options: {
|
|
102
|
+
dispatchKey?: string;
|
|
100
103
|
submission?: RunSubmission;
|
|
101
104
|
agent: AgentPort;
|
|
102
105
|
store: RunStore;
|
|
@@ -197,6 +197,7 @@ async function preparePipeline(options) {
|
|
|
197
197
|
namedSecrets,
|
|
198
198
|
});
|
|
199
199
|
run = await options.store.createRun({
|
|
200
|
+
dispatchKey: options.dispatchKey,
|
|
200
201
|
submission: options.submission,
|
|
201
202
|
runId,
|
|
202
203
|
pipelineId: loaded.pipeline.id,
|
|
@@ -299,6 +300,7 @@ export async function startPipeline(options) {
|
|
|
299
300
|
const cwd = options.cwd ?? process.cwd();
|
|
300
301
|
const projectRoot = options.projectRoot ?? cwd;
|
|
301
302
|
const prepared = await preparePipeline({
|
|
303
|
+
dispatchKey: options.dispatchKey,
|
|
302
304
|
submission: options.submission,
|
|
303
305
|
agent: options.agent,
|
|
304
306
|
store: options.store,
|
|
@@ -327,6 +329,15 @@ export async function startPipeline(options) {
|
|
|
327
329
|
callerId: options.callerId,
|
|
328
330
|
...(options.skills !== undefined ? { skills: options.skills } : {}),
|
|
329
331
|
});
|
|
332
|
+
if (prepared.run.created === false) {
|
|
333
|
+
const meta = await prepared.store.readRunMeta(prepared.run.runId);
|
|
334
|
+
return {
|
|
335
|
+
created: false,
|
|
336
|
+
runId: prepared.run.runId,
|
|
337
|
+
runDir: prepared.run.workspaceDir,
|
|
338
|
+
done: Promise.resolve({ ok: meta.status === "succeeded", outcome: meta.status === "succeeded" ? "succeeded" : "failed", runId: prepared.run.runId, runDir: prepared.run.workspaceDir, reason: "Recovered existing dispatch; execution was not restarted" }),
|
|
339
|
+
};
|
|
340
|
+
}
|
|
330
341
|
const done = executeStages(prepared, {
|
|
331
342
|
maxActiveStagesPerRun: options.maxActiveStagesPerRun,
|
|
332
343
|
executionMode: prepared.executionMode,
|