stageflow 0.40.0 → 0.44.0

package/dist/evals/types.d.ts

@@ -0,0 +1,357 @@
+export type FieldOp = "exists" | "not_empty" | "equals" | "not_equals" | "contains" | "matches" | "gte" | "lte" | "between" | "in" | "length_gte" | "length_lte";
+export type FieldCheck = {
+    path: string;
+    op: FieldOp;
+    value?: unknown;
+};
+export type RubricExample = {
+    verdict: "pass" | "fail";
+    output: string;
+    reason?: string;
+} | {
+    score: number;
+    output: string;
+    reason?: string;
+};
+export type EvalType = "capability" | "regression";
+export type StageEvalTarget = {
+    kind: "stage";
+    pipeline?: string;
+    stage?: string;
+    file?: string;
+};
+export type AgentEvalTarget = {
+    kind: "agent";
+    agent: string;
+};
+export type InternalAgentEvalTarget = {
+    kind: "internal_agent";
+    profile: string;
+};
+export type EvalTarget = StageEvalTarget | AgentEvalTarget | InternalAgentEvalTarget;
+/** A conversation eval: a project agent or a built-in operator agent. */
+export declare function isConversationTarget(kind: string): boolean;
+export type ConversationMode = "script" | "simulated";
+export type ConversationTools = "real" | "read_only" | "mocked";
+export type ConversationApprovals = "deny" | "auto_approve_read" | "record_only";
+export type EvalConversation = {
+    mode?: ConversationMode;
+    max_turns?: number;
+    turn_timeout_ms?: number;
+    tools?: ConversationTools;
+    approvals?: ConversationApprovals;
+    read_only_tools?: string[];
+    allow_real_tools?: boolean;
+};
+export type ResolvedConversation = {
+    mode: ConversationMode;
+    maxTurns: number;
+    turnTimeoutMs: number;
+    tools: ConversationTools;
+    approvals: ConversationApprovals;
+    readOnlyTools: string[];
+    allowRealTools: boolean;
+};
+export type EvalTask = {
+    goal?: string;
+    context?: string;
+    constraints?: string;
+};
+export type EvalDefaults = {
+    repeats?: number;
+    judge?: string;
+    pass_threshold?: number;
+    max_cost_usd?: number;
+    concurrency?: number;
+    model?: string;
+    user_model?: string;
+};
+export type EvalBudget = {
+    latency_ms?: number;
+    cost_usd?: number;
+};
+type GraderCommon = {
+    id: string;
+    description?: string;
+    weight?: number;
+    must_pass?: boolean;
+};
+export type ToolArgCheck = {
+    tool: string;
+    path: string;
+    op: FieldOp;
+    value?: unknown;
+};
+export type FinalReplyCheck = {
+    op: FieldOp;
+    value?: unknown;
+};
+export type GraderConfig = (GraderCommon & {
+    type: "schema";
+}) | (GraderCommon & {
+    type: "field";
+    checks: FieldCheck[];
+}) | (GraderCommon & {
+    type: "reference";
+    mode: "each_mentioned" | "equals";
+    path?: string;
+    expected: string;
+    pass_at?: number;
+}) | (GraderCommon & {
+    type: "rubric";
+    rubric: string;
+    scale?: "pass_fail" | "1_to_5";
+    pass_at?: number;
+    judge?: string;
+    examples?: RubricExample[];
+}) | (GraderCommon & {
+    type: "code";
+    code: string;
+}) | (GraderCommon & {
+    type: "tool_calls";
+    must_call?: string[];
+    must_not_call?: string[];
+    must_ask_first?: string[];
+    max_calls?: number;
+    args?: ToolArgCheck[];
+}) | (GraderCommon & {
+    type: "conversation_check";
+    max_turns?: number;
+    max_agent_questions_per_turn?: number;
+    final_reply?: FinalReplyCheck[];
+}) | (GraderCommon & {
+    type: "goal_reached";
+    judge?: string;
+    guidance?: string;
+});
+export type EvalFile = {
+    version: 1;
+    id: string;
+    name?: string;
+    description?: string;
+    type?: EvalType;
+    target: EvalTarget;
+    dataset: string;
+    task?: EvalTask;
+    defaults?: EvalDefaults;
+    conversation?: EvalConversation;
+    graders: GraderConfig[];
+    budget?: EvalBudget;
+};
+export type StageTargetRef = {
+    kind: "stage";
+    pipelinePath?: string;
+    stageId?: string;
+    file?: string;
+};
+export type AgentTargetRef = {
+    kind: "agent";
+    agentId: string;
+    revision?: number;
+};
+export type InternalAgentTargetRef = {
+    kind: "internal_agent";
+    profileId: string;
+};
+export type TargetRef = StageTargetRef | AgentTargetRef | InternalAgentTargetRef;
+type ResolvedGraderCommon = {
+    id: string;
+    weight: number;
+    mustPass: boolean;
+    hash: string;
+    description?: string;
+};
+export type ResolvedGrader = (ResolvedGraderCommon & {
+    type: "schema";
+}) | (ResolvedGraderCommon & {
+    type: "field";
+    checks: FieldCheck[];
+}) | (ResolvedGraderCommon & {
+    type: "reference";
+    mode: "each_mentioned" | "equals";
+    path: string;
+    expected: string;
+    passAt: number;
+}) | (ResolvedGraderCommon & {
+    type: "rubric";
+    rubric: string;
+    scale: "pass_fail" | "1_to_5";
+    passAt: number;
+    judge: string;
+    examples: RubricExample[];
+}) | (ResolvedGraderCommon & {
+    type: "code";
+    code: string;
+}) | (ResolvedGraderCommon & {
+    type: "tool_calls";
+    mustCall?: string[];
+    mustNotCall?: string[];
+    mustAskFirst?: string[];
+    maxCalls?: number;
+    args: ToolArgCheck[];
+}) | (ResolvedGraderCommon & {
+    type: "conversation_check";
+    maxTurns?: number;
+    maxAgentQuestionsPerTurn?: number;
+    finalReply: FinalReplyCheck[];
+}) | (ResolvedGraderCommon & {
+    type: "goal_reached";
+    judge: string;
+    guidance?: string;
+});
+export type PlanCase = {
+    id: string;
+    position: number;
+    title?: string;
+    goal?: string;
+    input: Record<string, unknown>;
+    expected?: unknown;
+    tags: string[];
+    metadata?: unknown;
+};
+export type EvalPlan = {
+    evalRunId: string;
+    projectRoot: string;
+    evalId: string;
+    target: TargetRef;
+    targetVersionHash: string;
+    outputSchema?: unknown;
+    stageModel: string;
+    task: {
+        goal?: string;
+        context?: string;
+        constraints?: string;
+    };
+    cases: PlanCase[];
+    graders: ResolvedGrader[];
+    repeats: number;
+    passThreshold: number;
+    concurrency: number;
+    maxCostUsd: number;
+    modelOverride?: string;
+    budget?: {
+        latencyMs?: number;
+        costUsd?: number;
+    };
+    /** Agent plans only: the revision pinned at plan time and the conversation settings. */
+    agent?: {
+        definitionHash: string;
+        definition: Record<string, unknown>;
+    };
+    conversation?: ResolvedConversation;
+    /** Agent plans only: null when no selected case is simulated. */
+    userModel?: string | null;
+};
+export type EnvelopeOutput = {
+    kind: "envelope";
+    status: "success" | "failure";
+    summary: string;
+    payload?: Record<string, unknown>;
+    artifacts: string[];
+};
+export type ConversationStopReason = "script_end" | "resolved" | "abandoned" | "max_turns" | "no_progress" | "empty_message" | "error" | "timeout";
+export type ConversationMessage = {
+    role: "user" | "assistant";
+    text: string;
+    turn: number;
+};
+export type ConversationToolCall = {
+    id: string;
+    turn: number;
+    name: string;
+    args: Record<string, unknown>;
+    result_summary: string;
+    status: "ok" | "error" | "not_run";
+    policy_outcome: "ran" | "stubbed" | "mocked" | "would_ask_first" | "denied";
+    session_id?: string;
+};
+export type ConversationUsage = {
+    cost_usd: number;
+    agent_cost_usd: number;
+    simulator_cost_usd: number;
+    tokens_in: number;
+    tokens_out: number;
+    simulator_tokens_in: number;
+    simulator_tokens_out: number;
+    simulator_estimated: boolean;
+    agent_calls: number;
+    simulator_calls: number;
+};
+/** The stored JSON shape (snake_case), which graders, the judge renderer and the console read as is. */
+export type ConversationOutput = {
+    kind: "conversation";
+    status: "success" | "failure";
+    summary: string;
+    messages: ConversationMessage[];
+    tool_calls: ConversationToolCall[];
+    final_reply: string;
+    turns: number;
+    stop_reason: ConversationStopReason;
+    usage: ConversationUsage;
+    truncated?: Record<string, number>;
+    /** Transport only: the bounded rendered transcript for the judge. The manager drops it before storing. */
+    transcript?: string;
+};
+export type CaseOutput = EnvelopeOutput | ConversationOutput;
+export type RunCaseRequest = {
+    evalRunId: string;
+    caseId: string;
+    repeatIndex: number;
+};
+export type RunCaseFailureCode = "EVAL_STAGE_FAILED" | "EVAL_STAGE_NEEDS_INPUT" | "EVAL_STAGE_TIMEOUT" | "EVAL_BUDGET_EXCEEDED" | "EVAL_CANCELLED" | "EVAL_ENGINE_ERROR" | "EVAL_AGENT_FAILED" | "EVAL_AGENT_EMPTY_REPLY" | "EVAL_AGENT_TIMEOUT" | "EVAL_SIMULATOR_ERROR";
+export type RunCaseResult = {
+    ok: true;
+    /** Root subject id: the stage run id (set by the manager when absent) or the root agent session id. */
+    subjectRef?: string;
+    stageRunId?: string;
+    output: CaseOutput;
+    latencyMs: number;
+    costUsd: number;
+    tokensIn: number;
+    tokensOut: number;
+} | {
+    ok: false;
+    code: RunCaseFailureCode;
+    message: string;
+    subjectRef?: string;
+    stageRunId?: string;
+    output?: ConversationOutput;
+    latencyMs?: number;
+    costUsd?: number;
+};
+export type NormalizedGrade = {
+    graderId: string;
+    graderType: string;
+    status: "graded" | "error";
+    passed?: boolean;
+    score?: number;
+    rawValue?: number;
+    reason?: string;
+    errorCode?: string;
+    details?: unknown;
+    judgeModel?: string;
+    judgeCostUsd?: number;
+    judgeTokensIn?: number;
+    judgeTokensOut?: number;
+};
+export type NormalizedResult = {
+    caseId: string;
+    repeatIndex: number;
+    run: RunCaseResult;
+    grades: NormalizedGrade[];
+};
+export interface EngineIo {
+    runCase(req: RunCaseRequest): Promise<RunCaseResult>;
+    onResult(result: NormalizedResult): void;
+    reportJudgeCost(usd: number): void;
+    onPhase(caseId: string, repeatIndex: number, phase: "stage_running" | "agent_running" | "grading" | "done"): void;
+    log(level: "info" | "warn" | "error", message: string): void;
+}
+export interface EvalEngine {
+    readonly name: string;
+    version(): Promise<string>;
+    /** Throws an Error with a `code` when the plan can never run on this engine (checked before a run row exists). */
+    validatePlan?(plan: EvalPlan): void;
+    run(plan: EvalPlan, io: EngineIo, signal: AbortSignal): Promise<void>;
+}
+export {};

package/dist/evals/types.js

@@ -0,0 +1,4 @@
+/** A conversation eval: a project agent or a built-in operator agent. */
+export function isConversationTarget(kind) {
+    return kind === "agent" || kind === "internal_agent";
+}

package/dist/guide/cardContract.d.ts

@@ -0,0 +1,133 @@
+export declare const GUIDE_LINK_KINDS: readonly ["pipeline", "stage", "task", "agent", "trigger", "eval", "eval_run", "dataset", "connection", "memory", "run"];
+export declare const GUIDE_RESOURCE_KINDS: readonly ["agent", "trigger", "eval", "connection", "memory", "task"];
+export declare const GUIDE_UI_COMPONENTS: readonly ["pipeline", "resource", "plan", "choices", "eval_report", "links"];
+export declare const GUIDE_UI_MAX_TEXT = 300;
+export declare const GUIDE_UI_MAX_PLAN_STEPS = 12;
+export declare const GUIDE_UI_MAX_CHOICES = 6;
+export declare const GUIDE_UI_MAX_LINKS = 8;
+export declare const GUIDE_UI_MAX_FAILURES = 5;
+export declare const GUIDE_UI_MAX_MESSAGES = 5;
+export type GuideLinkKind = (typeof GUIDE_LINK_KINDS)[number];
+export type GuideUiComponent = (typeof GUIDE_UI_COMPONENTS)[number];
+export type GuideLink = {
+    kind: GuideLinkKind;
+    id: string;
+    path?: string;
+    projectRoot?: string;
+    label?: string;
+};
+export type GuidePipelineStage = {
+    id: string;
+    summary?: string;
+    model?: string;
+    needs: string[];
+    gates: string[];
+    channels: string[];
+    hasVerify: boolean;
+};
+export type GuideUi = {
+    component: "pipeline";
+    title: string;
+    pipeline: {
+        id: string;
+        path: string;
+        model?: string;
+        stages: GuidePipelineStage[];
+        memories: string[];
+        task?: {
+            id: string;
+            goal: string;
+        };
+    };
+    validation: {
+        ok: boolean;
+        errors: number;
+        warnings: number;
+        messages: string[];
+    };
+    link: GuideLink;
+} | {
+    component: "resource";
+    title: string;
+    kind: GuideLinkKind;
+    fields: Array<{
+        label: string;
+        value: string;
+    }>;
+    status?: string;
+    link: GuideLink;
+} | {
+    component: "plan";
+    title: string;
+    steps: Array<{
+        title: string;
+        detail?: string;
+        status: "todo" | "current" | "done";
+    }>;
+} | {
+    component: "choices";
+    question: string;
+    options: Array<{
+        id: string;
+        label: string;
+        description?: string;
+    }>;
+    allowOther: boolean;
+} | {
+    component: "eval_report";
+    title: string;
+    evalId: string;
+    runId: string;
+    status: string;
+    passRate: number | null;
+    passed: boolean | null;
+    cases: number;
+    failures: Array<{
+        caseId: string;
+        reason: string;
+    }>;
+    costUsd: number | null;
+    link: GuideLink;
+} | {
+    component: "links";
+    title?: string;
+    links: GuideLink[];
+};
+export type GuideChoicesUi = Extract<GuideUi, {
+    component: "choices";
+}>;
+export type GuidePlanUi = Extract<GuideUi, {
+    component: "plan";
+}>;
+export type GuideLinksUi = Extract<GuideUi, {
+    component: "links";
+}>;
+export declare class GuideCardError extends Error {
+    readonly code: string;
+    constructor(code: string);
+}
+/** Structural check for a Guide card payload. Returns the same value when valid. */
+export declare function validateGuideUi(value: unknown): GuideUi | null;
+/** Card payload from a `show_ui` / confirm tool result, or null when the console cannot draw it. */
+export declare function readGuideUi(result: unknown): GuideUi | null;
+export declare function buildChoicesUi(input: {
+    question: string;
+    options: Array<{
+        id: string;
+        label: string;
+        description?: string;
+    }>;
+    allowOther?: boolean;
+}): GuideChoicesUi;
+export declare function buildPlanUi(input: {
+    title: string;
+    steps: Array<{
+        title: string;
+        detail?: string;
+        status: "todo" | "current" | "done";
+    }>;
+}): GuidePlanUi;
+export declare function buildLinksUi(input: {
+    title?: string;
+    links: GuideLink[];
+}): GuideLinksUi;

package/dist/guide/cardContract.js

@@ -0,0 +1,156 @@
+export const GUIDE_LINK_KINDS = [
+    "pipeline",
+    "stage",
+    "task",
+    "agent",
+    "trigger",
+    "eval",
+    "eval_run",
+    "dataset",
+    "connection",
+    "memory",
+    "run",
+];
+export const GUIDE_RESOURCE_KINDS = ["agent", "trigger", "eval", "connection", "memory", "task"];
+export const GUIDE_UI_COMPONENTS = ["pipeline", "resource", "plan", "choices", "eval_report", "links"];
+export const GUIDE_UI_MAX_TEXT = 300;
+export const GUIDE_UI_MAX_PLAN_STEPS = 12;
+export const GUIDE_UI_MAX_CHOICES = 6;
+export const GUIDE_UI_MAX_LINKS = 8;
+export const GUIDE_UI_MAX_FAILURES = 5;
+export const GUIDE_UI_MAX_MESSAGES = 5;
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function hasArrays(value, keys) {
+    return isRecord(value) && keys.every((key) => Array.isArray(value[key]));
+}
+function isLinkKind(value) {
+    return typeof value === "string" && GUIDE_LINK_KINDS.includes(value);
+}
+function clipText(value) {
+    const trimmed = value.trim();
+    if (!trimmed)
+        return "";
+    return trimmed.length > GUIDE_UI_MAX_TEXT ? `${trimmed.slice(0, GUIDE_UI_MAX_TEXT - 1)}…` : trimmed;
+}
+function requireText(value, code = "guide.ui_invalid_args") {
+    const text = clipText(value);
+    if (!text)
+        throw new GuideCardError(code);
+    return text;
+}
+export class GuideCardError extends Error {
+    code;
+    constructor(code) {
+        super(code);
+        this.code = code;
+    }
+}
+/** Structural check for a Guide card payload. Returns the same value when valid. */
+export function validateGuideUi(value) {
+    if (!isRecord(value) || typeof value.component !== "string")
+        return null;
+    if (!GUIDE_UI_COMPONENTS.includes(value.component))
+        return null;
+    switch (value.component) {
+        case "pipeline": {
+            if (!hasArrays(value.pipeline, ["stages"]) || !isRecord(value.validation) || !isRecord(value.link))
+                return null;
+            const stages = value.pipeline.stages;
+            if (!stages.every((stage) => isRecord(stage) && typeof stage.id === "string"))
+                return null;
+            break;
+        }
+        case "resource":
+            if (!Array.isArray(value.fields) || !isRecord(value.link))
+                return null;
+            break;
+        case "plan":
+            if (!Array.isArray(value.steps))
+                return null;
+            break;
+        case "choices":
+            if (typeof value.question !== "string" || !Array.isArray(value.options))
+                return null;
+            if (!value.options.every((option) => isRecord(option) && typeof option.id === "string" && typeof option.label === "string")) {
+                return null;
+            }
+            break;
+        case "eval_report":
+            if (!Array.isArray(value.failures) || !isRecord(value.link))
+                return null;
+            break;
+        case "links":
+            if (!Array.isArray(value.links))
+                return null;
+            break;
+        default:
+            return null;
+    }
+    return value;
+}
+/** Card payload from a `show_ui` / confirm tool result, or null when the console cannot draw it. */
+export function readGuideUi(result) {
+    if (!isRecord(result))
+        return null;
+    return validateGuideUi(result.ui);
+}
+export function buildChoicesUi(input) {
+    const question = requireText(input.question);
+    if (input.options.length < 1 || input.options.length > GUIDE_UI_MAX_CHOICES) {
+        throw new GuideCardError("guide.ui_invalid_args");
+    }
+    const options = input.options.map((option) => {
+        const id = requireText(option.id);
+        const label = requireText(option.label);
+        const description = option.description !== undefined ? clipText(option.description) : undefined;
+        return description ? { id, label, description } : { id, label };
+    });
+    return {
+        component: "choices",
+        question,
+        options,
+        allowOther: input.allowOther ?? false,
+    };
+}
+export function buildPlanUi(input) {
+    const title = requireText(input.title);
+    if (input.steps.length < 1 || input.steps.length > GUIDE_UI_MAX_PLAN_STEPS) {
+        throw new GuideCardError("guide.ui_invalid_args");
+    }
+    return {
+        component: "plan",
+        title,
+        steps: input.steps.map((step) => {
+            const detail = step.detail !== undefined ? clipText(step.detail) : undefined;
+            return {
+                title: requireText(step.title),
+                ...(detail ? { detail } : {}),
+                status: step.status,
+            };
+        }),
+    };
+}
+export function buildLinksUi(input) {
+    if (input.links.length < 1 || input.links.length > GUIDE_UI_MAX_LINKS) {
+        throw new GuideCardError("guide.ui_invalid_args");
+    }
+    const title = input.title !== undefined ? requireText(input.title) : undefined;
+    return {
+        component: "links",
+        ...(title ? { title } : {}),
+        links: input.links.map((link) => {
+            if (!isLinkKind(link.kind))
+                throw new GuideCardError("guide.ui_invalid_args");
+            const next = { kind: link.kind, id: requireText(link.id) };
+            if (link.path)
+                next.path = requireText(link.path);
+            if (link.projectRoot)
+                next.projectRoot = requireText(link.projectRoot);
+            if (link.label)
+                next.label = requireText(link.label);
+            return next;
+        }),
+    };
+}

package/dist/guide/chat.d.ts

@@ -0,0 +1,45 @@
+import { type OperatorAgentHost } from "../operatorAgent/index.js";
+import { type GuideMessage } from "./threadStore.js";
+export declare const GUIDE_MAX_MESSAGE_CHARS = 8000;
+export declare class GuideError extends Error {
+    readonly code: string;
+    readonly status: number;
+    constructor(code: string, status: number);
+}
+export declare const GUIDE_TOOL_RESULT_MAX_BYTES = 20000;
+export type GuideToolFrame = {
+    type: "tool";
+    id: string;
+    name: string;
+    status: "running" | "complete" | "error";
+    args?: Record<string, unknown>;
+    result?: unknown;
+    error?: string;
+};
+export type GuideFrame = {
+    type: "delta";
+    text: string;
+} | GuideToolFrame | {
+    type: "done";
+    message: GuideMessage;
+} | {
+    type: "error";
+    code: string;
+    message: string;
+};
+export declare function resetGuideSessionsForTests(): void;
+export declare function guideAgentInfo(host: OperatorAgentHost): {
+    name: string;
+    greeting: string;
+};
+export declare function isGuideThreadRunning(threadId: string): boolean;
+export declare function abortGuideTurn(threadId: string): boolean;
+export declare function streamGuideTurn(input: {
+    host: OperatorAgentHost;
+    threadId: string;
+    text: string;
+    attachments?: unknown;
+    model?: string | null;
+    storeRoot?: string;
+    signal?: AbortSignal;
+}): AsyncGenerator<GuideFrame>;