pi-extensible-workflows 0.3.0

package/dist/src/workflow-evals.d.ts

@@ -0,0 +1,290 @@
+export { resolveWorkflowSkillPath } from "./eval-capture-extension.js";
+import { type AgentIdentity, type JsonSchema, type JsonValue, type StaticWorkflowCall, type StaticWorkflowExecution, type WorkflowErrorCode } from "./index.js";
+export type SignificantAction = {
+    kind: "tool";
+    name: string;
+} | {
+    kind: "text";
+} | {
+    kind: "thinking";
+};
+export type SequenceExpectation = readonly string[] | {
+    equals?: readonly string[];
+    startsWith?: readonly string[];
+};
+export type JsonResultType = "null" | "boolean" | "number" | "integer" | "string" | "array" | "object";
+export interface JsonResultShape {
+    type?: JsonResultType;
+    equals?: JsonValue;
+    nonEmpty?: boolean;
+    requiredKeys?: readonly string[];
+    propertyTypes?: Readonly<Record<string, JsonResultType>>;
+    forbiddenProperties?: readonly string[];
+    count?: number;
+    minCount?: number;
+    properties?: Readonly<Record<string, JsonResultShape>>;
+}
+export interface ExpectedReplayResult {
+    workflowIndex?: number;
+    equals?: JsonValue;
+    match?: JsonResultShape;
+}
+export interface OutputSchemaShape {
+    type?: string;
+    requiredKeys?: readonly string[];
+    propertyTypes?: Readonly<Record<string, string>>;
+    forbiddenProperties?: readonly string[];
+    count?: number;
+    minCount?: number;
+}
+export interface AgentPolicyExpectation {
+    callIndex: number;
+    role?: string;
+    model?: string;
+    forbidOptions?: readonly ("role" | "model" | "thinking" | "tools" | "retries")[];
+    tools?: {
+        mode: "omitted" | "empty" | "exact";
+        values?: readonly string[];
+    };
+}
+export interface AgentStructureExpectation {
+    execution: StaticWorkflowExecution;
+    operation?: "parallel" | "pipeline";
+    agents: readonly AgentOrderExpectation[];
+}
+export interface AgentOrderExpectation {
+    role?: string;
+    model?: string;
+    promptIncludes?: string;
+    execution?: StaticWorkflowExecution;
+}
+export interface DataFlowExpectation {
+    binding: string;
+    toAgentIndex: number;
+}
+export interface ParentAssistantBatch {
+    index: number;
+    parts: readonly JsonValue[];
+    tools: readonly string[];
+    usage?: ParentUsage;
+}
+export interface ParentToolResult {
+    toolCallId?: string;
+    details?: JsonValue;
+    isError?: boolean;
+    text?: string;
+}
+export interface ParentUsage {
+    input: number;
+    output: number;
+    cacheRead: number;
+    cacheWrite: number;
+    totalTokens: number;
+    cost: number;
+    models: readonly {
+        model: string;
+        cost: number;
+    }[];
+}
+export interface ParentOracle {
+    assistantBatches: readonly ParentAssistantBatch[];
+    workflowToolResults: readonly ParentToolResult[];
+    skillReads: readonly string[];
+    firstSignificantAction?: SignificantAction;
+    firstTool?: string;
+    firstBatchToolSequence: readonly string[];
+    toolsBeforeFirstWorkflow: readonly string[];
+    firstWorkflowBatchToolSequence: readonly string[];
+    parentToolSequence: readonly string[];
+    workflowCallCount: number;
+    usage: ParentUsage;
+}
+export interface CapturedWorkflowCall {
+    batch: number;
+    toolCallId?: string;
+    arguments: JsonValue;
+    script?: string;
+}
+export interface EvalExpectations {
+    firstSignificantAction?: SignificantAction;
+    firstTool?: string;
+    firstBatchToolSequence?: SequenceExpectation;
+    parentToolSequence?: SequenceExpectation;
+    workflowCallCount?: number | {
+        min?: number;
+        max?: number;
+    };
+    requiredOperations?: readonly StaticWorkflowCall["kind"][];
+    forbiddenOperations?: readonly StaticWorkflowCall["kind"][];
+    requiredRoles?: readonly string[];
+    minimumAgentCalls?: number;
+    requireOutputSchema?: OutputSchemaShape | boolean;
+    expectedResults?: readonly ExpectedReplayResult[];
+    agentPolicies?: readonly AgentPolicyExpectation[];
+    requiredAgentOrder?: readonly AgentOrderExpectation[];
+    requiredAgentStructures?: readonly AgentStructureExpectation[];
+    requiredDataFlow?: readonly DataFlowExpectation[];
+}
+export interface SemanticCriterion {
+    id: string;
+    description: string;
+}
+export interface WorkflowEvalCase {
+    id: string;
+    prompt: string;
+    timeoutMs?: number;
+    maxCost: number;
+    expectations: EvalExpectations;
+    expectedWorkflowCalls?: number;
+    semanticCriteria?: readonly SemanticCriterion[];
+}
+export interface ReplayAgentCall {
+    prompt: string;
+    options: Readonly<Record<string, JsonValue>>;
+    identity: AgentIdentity;
+}
+export interface ReplayTrace {
+    agentCalls: readonly ReplayAgentCall[];
+    phases: readonly string[];
+    logs: readonly string[];
+    maxConcurrentAgents: number;
+}
+export interface ReplayReport {
+    script: string;
+    result?: JsonValue;
+    trace?: ReplayTrace;
+    error?: string;
+}
+export interface ProductionValidationReport {
+    callIndex: number;
+    valid: boolean;
+    errorCode?: WorkflowErrorCode;
+    message?: string;
+}
+export interface CriterionResult {
+    id: string;
+    pass: boolean;
+    evidence: string;
+}
+export interface StaticCandidateReport {
+    callIndices: readonly number[];
+    criteria: readonly CriterionResult[];
+    passed: boolean;
+}
+export interface SemanticJudgeReport {
+    criteria: readonly CriterionResult[];
+    usage: ParentUsage;
+    raw: string;
+}
+export type EvalAccounting = ParentUsage;
+export interface EvalMetrics {
+    parentUsageThroughCandidate: ParentUsage | null;
+    parentOutputTokensThroughCandidate: number | null;
+    nonWorkflowToolSequenceBeforeCandidate: readonly string[];
+    nonWorkflowToolCallCountBeforeCandidate: number;
+    workflowCallCountBeforeCandidate: number;
+    invalidWorkflowCallCount: number;
+    productionValidationErrorCodes: readonly string[];
+    candidateCallIndices: readonly number[];
+    staticCandidates: readonly StaticCandidateReport[];
+    semanticCriteria: readonly CriterionResult[];
+    anyValidCandidate: boolean;
+    requiredWorkflowCallCount: number;
+    surplusWorkflowCallCount: number;
+}
+export interface EvalCaseResult {
+    id: string;
+    status: "passed" | "failed" | "timed_out" | "budget_exceeded" | "skipped";
+    limits: {
+        timeoutMs?: number;
+        maxCost: number;
+    };
+    oracle?: ParentOracle;
+    workflows: readonly CapturedWorkflowCall[];
+    productionValidation: readonly ProductionValidationReport[];
+    semanticJudge?: SemanticJudgeReport;
+    metrics: EvalMetrics;
+    accounting: EvalAccounting;
+    accountingTrustworthy: boolean;
+    diagnostics: readonly string[];
+    errors: readonly string[];
+    cleanup: {
+        processExited: boolean;
+        processGroupTerminated: boolean;
+        tempRootRemoved: boolean;
+        captureIdentityVerified: boolean;
+        realWorkflowAgentsLaunched: number | null;
+    };
+}
+export declare const SAFE_PARENT_EVAL_TOOLS: readonly ["read", "grep", "find", "bash", "workflow"];
+export declare function validateWorkflowEvalCases(values: readonly unknown[], source?: string): readonly WorkflowEvalCase[];
+export declare function loadWorkflowEvalCases(directory?: string): readonly WorkflowEvalCase[];
+export declare const INITIAL_WORKFLOW_EVAL_CASES: readonly WorkflowEvalCase[];
+export declare function matchesJsonResult(shape: JsonResultShape, value: JsonValue): boolean;
+export declare function matchesOutputSchema(shape: OutputSchemaShape, schema: JsonSchema): boolean;
+export declare function extractParentOracle(entries: readonly unknown[]): ParentOracle;
+export declare function extractParentOracleFile(path: string): ParentOracle;
+export declare function extractCapturedWorkflows(oracle: ParentOracle): CapturedWorkflowCall[];
+export declare function captureValidationReports(oracle: ParentOracle, calls: readonly CapturedWorkflowCall[]): {
+    reports: ProductionValidationReport[];
+    errors: string[];
+    verified: boolean;
+};
+export declare function evalExpectationErrors(oracle: ParentOracle, expectations: EvalExpectations): string[];
+export declare function replayExpectationErrors(calls: readonly CapturedWorkflowCall[], reports: readonly ReplayReport[], expectations: EvalExpectations): string[];
+export declare function staticExpectationResults(calls: readonly CapturedWorkflowCall[], expectations: EvalExpectations): CriterionResult[];
+export declare function selectStaticCandidate(calls: readonly CapturedWorkflowCall[], validations: readonly ProductionValidationReport[], expectations: EvalExpectations, requiredCount?: number): {
+    callIndices: readonly number[];
+    reports: readonly StaticCandidateReport[];
+};
+export declare function assertEvalScriptSafe(script: string): void;
+export declare function matchesJsonSchema(schema: JsonSchema, value: JsonValue): boolean;
+export declare function replayWorkflowScript(script: string, args?: JsonValue, signal?: AbortSignal): Promise<{
+    result: JsonValue;
+    trace: ReplayTrace;
+}>;
+export interface CaptureCaseInput {
+    case: WorkflowEvalCase;
+    model: string;
+    provider?: string;
+    thinking?: string;
+    piCommand?: string;
+    maxCost: number;
+}
+export declare function parseSemanticJudge(raw: string, criteria: readonly SemanticCriterion[]): CriterionResult[];
+export declare function captureEvalCase(input: CaptureCaseInput): Promise<EvalCaseResult>;
+export interface IsolatedProcessOptions {
+    childPath: string;
+    timeoutMs?: number;
+    env?: NodeJS.ProcessEnv;
+    onStderr?: (chunk: string) => void;
+}
+export interface IsolatedProcessResult<T> {
+    value?: T;
+    timedOut: boolean;
+    exitCode: number | null;
+    processGroupTerminated: boolean;
+    stderr: string;
+    error?: string;
+}
+export declare function runIsolatedProcess<T>(payload: unknown, options: IsolatedProcessOptions): Promise<IsolatedProcessResult<T>>;
+export interface WorkflowEvalRunOptions {
+    cases?: readonly WorkflowEvalCase[];
+    caseIds?: readonly string[];
+    model?: string;
+    provider?: string;
+    thinking?: string;
+    piCommand?: string;
+    timeoutMs?: number;
+    spendCeiling?: number;
+    artifactsDir?: string;
+    onProgress?: (message: string) => void;
+}
+export interface WorkflowEvalRunResult {
+    artifactDir: string;
+    cases: readonly EvalCaseResult[];
+    spent: number;
+    skipped: readonly string[];
+}
+export declare function runWorkflowEvals(options?: WorkflowEvalRunOptions): Promise<WorkflowEvalRunResult>;
+export declare function formatEvalSummary(result: WorkflowEvalRunResult): string;