sammy-sdk 0.1.0

package/dist/eval/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IACnC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IACvC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IACtC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CACjC,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAsB,EACtB,aAAqB,EACrB,SAAmE,EACnE,UAAsB,EACtB,KAAuB;IAEvB,MAAM,MAAM,GAAG;;;QAGT,QAAQ,CAAC,IAAI;gBACL,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;qBAC/D,QAAQ,CAAC,gBAAgB,IAAI,mBAAmB;;;EAGnE,aAAa;;;EAGb,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;;;;;;;;;;;;;;;;;;;EAmBzG,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC;QACtC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK;QAC5B,OAAO,EAAE,OAAO;QAChB,QAAQ,EAAE;YACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,oGAAoG,EAAE;YACjI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;SAClC;QACD,WAAW,EAAE,UAAU,CAAC,KAAK,CAAC,WAAW;QACzC,cAAc,EAAE,MAAM;KACvB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,OAAO,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;YACnE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,gCAAgC,EAAE,WAAW,EAAE,EAAE;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,OAAqB,EACrB,QAAsB,EACtB,SAAiB,EACjB,WAAmB;IAEnB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,EAAE,aAAa,SAAS,EAAE,CAAC,CAAC;IACjG,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC5F,CAAC"}

package/dist/eval/loop-guard.d.ts

@@ -0,0 +1,12 @@
+import type { LoopState, LoopGuardConfig, EvaluationScorecard, TerminationReason, DimensionName } from "./types.js";
+export interface GuardResult {
+    shouldContinue: boolean;
+    reason?: TerminationReason;
+}
+export declare function checkGuards(state: LoopState, config: LoopGuardConfig): GuardResult;
+export declare function updateLockedDimensions(state: LoopState, scorecard: EvaluationScorecard): void;
+export declare function checkRegression(state: LoopState, scorecard: EvaluationScorecard, config: LoopGuardConfig): {
+    regressed: boolean;
+    dimension?: DimensionName;
+};
+//# sourceMappingURL=loop-guard.d.ts.map

package/dist/eval/loop-guard.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loop-guard.d.ts","sourceRoot":"","sources":["../../src/eval/loop-guard.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEpH,MAAM,WAAW,WAAW;IAC1B,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,GAAG,WAAW,CA4BlF;AAGD,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,SAAS,EAChB,SAAS,EAAE,mBAAmB,GAC7B,IAAI,CAMN;AAGD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,SAAS,EAChB,SAAS,EAAE,mBAAmB,EAC9B,MAAM,EAAE,eAAe,GACtB;IAAE,SAAS,EAAE,OAAO,CAAC;IAAC,SAAS,CAAC,EAAE,aAAa,CAAA;CAAE,CAWnD"}

package/dist/eval/loop-guard.js

@@ -0,0 +1,45 @@
+export function checkGuards(state, config) {
+    // Guard 1: Max iterations
+    if (state.currentIteration >= config.maxIterations) {
+        return { shouldContinue: false, reason: "max_iterations" };
+    }
+    // Guard 2: Diminishing returns
+    if (state.scoreHistory.length >= 2) {
+        const current = state.scoreHistory[state.scoreHistory.length - 1];
+        const previous = state.scoreHistory[state.scoreHistory.length - 2];
+        const improvement = (current.overallScore - previous.overallScore) * 100;
+        if (improvement < config.minImprovementPercent && improvement >= 0) {
+            return { shouldContinue: false, reason: "diminishing_returns" };
+        }
+    }
+    // Guard 5: Token budget
+    if (state.tokensUsed >= config.maxTokenBudget) {
+        return { shouldContinue: false, reason: "budget_exhausted" };
+    }
+    // Guard 6: All remaining issues need human intervention
+    if (state.escalations.length > 0 && state.appliedFixes.length === 0) {
+        return { shouldContinue: false, reason: "all_remaining_need_human" };
+    }
+    return { shouldContinue: true };
+}
+// Guard 3: Lock passing dimensions
+export function updateLockedDimensions(state, scorecard) {
+    for (const [dim, score] of Object.entries(scorecard.dimensions)) {
+        if (score.passed) {
+            state.lockedDimensions.add(dim);
+        }
+    }
+}
+// Guard 4: Regression detection
+export function checkRegression(state, scorecard, config) {
+    if (!config.enableRegressionRollback)
+        return { regressed: false };
+    for (const dim of state.lockedDimensions) {
+        const current = scorecard.dimensions[dim];
+        if (current && !current.passed) {
+            return { regressed: true, dimension: dim };
+        }
+    }
+    return { regressed: false };
+}
+//# sourceMappingURL=loop-guard.js.map

package/dist/eval/loop-guard.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loop-guard.js","sourceRoot":"","sources":["../../src/eval/loop-guard.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,WAAW,CAAC,KAAgB,EAAE,MAAuB;IACnE,0BAA0B;IAC1B,IAAI,KAAK,CAAC,gBAAgB,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACnD,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC7D,CAAC;IAED,+BAA+B;IAC/B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnE,MAAM,WAAW,GAAG,CAAC,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC;QAEzE,IAAI,WAAW,GAAG,MAAM,CAAC,qBAAqB,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;YACnE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;QAClE,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,IAAI,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC9C,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAC/D,CAAC;IAED,wDAAwD;IACxD,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACvE,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;AAClC,CAAC;AAED,mCAAmC;AACnC,MAAM,UAAU,sBAAsB,CACpC,KAAgB,EAChB,SAA8B;IAE9B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;QAChE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAoB,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;AACH,CAAC;AAED,gCAAgC;AAChC,MAAM,UAAU,eAAe,CAC7B,KAAgB,EAChB,SAA8B,EAC9B,MAAuB;IAEvB,IAAI,CAAC,MAAM,CAAC,wBAAwB;QAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAElE,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAC9B,CAAC"}

package/dist/eval/refiner.d.ts

@@ -0,0 +1,5 @@
+import type { SammyCloudClient } from "../cloud/sammy-cloud.js";
+import type { SammyConfig } from "../runtime/types.js";
+import type { DiagnosisReport, RefinementAction, AutoRefineConfig, LoopState } from "./types.js";
+export declare function applyRefinements(diagnosis: DiagnosisReport, config: SammyConfig, autoRefine: AutoRefineConfig, cloud: SammyCloudClient, state: LoopState, projectRoot: string): Promise<RefinementAction[]>;
+//# sourceMappingURL=refiner.d.ts.map

package/dist/eval/refiner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"refiner.d.ts","sourceRoot":"","sources":["../../src/eval/refiner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EACV,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,SAAS,EAC/D,MAAM,YAAY,CAAC;AAWpB,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,eAAe,EAC1B,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CA+C7B"}

package/dist/eval/refiner.js

@@ -0,0 +1,149 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import crypto from "node:crypto";
+const PRIORITY_ORDER = [
+    "SCHEMA_MISMATCH",
+    "TOOL_DESCRIPTION",
+    "PROMPT_CLARITY",
+    "ROUTING_AMBIGUITY",
+    "ARCHITECTURE",
+    "MODEL_CAPABILITY",
+];
+export async function applyRefinements(diagnosis, config, autoRefine, cloud, state, projectRoot) {
+    const actions = [];
+    // Sort root causes by priority
+    const sorted = [...diagnosis.rootCauses].sort((a, b) => {
+        const aIdx = PRIORITY_ORDER.indexOf(a.type);
+        const bIdx = PRIORITY_ORDER.indexOf(b.type);
+        return (aIdx === -1 ? 99 : aIdx) - (bIdx === -1 ? 99 : bIdx);
+    });
+    for (const cause of sorted) {
+        // Check if this fix type is allowed
+        if (!isAllowed(cause.type, autoRefine))
+            continue;
+        // Check for same-fix detection
+        const fixHash = crypto.createHash("sha256")
+            .update(`${cause.type}:${cause.tool || ""}:${cause.dimension}:${cause.suggestedFix}`)
+            .digest("hex").slice(0, 16);
+        if (state.skippedFixes.includes(fixHash))
+            continue;
+        const existingSameFixCount = state.appliedFixes.filter(f => crypto.createHash("sha256")
+            .update(`${f.rootCause}:${f.target.tool || ""}:${f.target.type}:${f.reason}`)
+            .digest("hex").slice(0, 16) === fixHash).length;
+        if (existingSameFixCount >= 2) {
+            state.skippedFixes.push(fixHash);
+            continue;
+        }
+        const action = await applyFix(cause, config, cloud, state.currentIteration, projectRoot);
+        if (action) {
+            actions.push(action);
+            state.appliedFixes.push(action);
+        }
+    }
+    // Save refinement history
+    const historyPath = path.join(projectRoot, ".sammy", "eval", "refinement-history.json");
+    const existing = fs.existsSync(historyPath) ? JSON.parse(fs.readFileSync(historyPath, "utf-8")) : [];
+    existing.push(...actions);
+    fs.mkdirSync(path.dirname(historyPath), { recursive: true });
+    fs.writeFileSync(historyPath, JSON.stringify(existing, null, 2));
+    return actions;
+}
+async function applyFix(cause, config, cloud, iteration, projectRoot) {
+    const timestamp = new Date().toISOString();
+    switch (cause.type) {
+        case "SCHEMA_MISMATCH": {
+            const domain = config.domains.find(d => d.tools.some(t => t.name === cause.tool));
+            const tool = domain?.tools.find(t => t.name === cause.tool);
+            if (!tool)
+                return null;
+            const before = JSON.stringify(tool.parameters);
+            // Use LLM to suggest tighter constraints
+            const response = await cloud.completion({
+                tier: "balanced",
+                purpose: "eval",
+                messages: [
+                    { role: "system", content: "You refine Zod parameter schemas. Return ONLY a JSON object with the improved parameters." },
+                    { role: "user", content: `Current parameters: ${before}\nIssue: ${cause.detail}\nSuggestion: ${cause.suggestedFix}\n\nReturn the improved parameters JSON.` },
+                ],
+                temperature: 0,
+                responseFormat: "json",
+            });
+            try {
+                const improved = JSON.parse(response.content);
+                tool.parameters = improved;
+                saveConfig(config, projectRoot);
+                return { iteration, timestamp, rootCause: cause.type, target: { type: "schema", tool: cause.tool }, before, after: JSON.stringify(improved), reason: cause.suggestedFix, triggeredBy: cause.scenario };
+            }
+            catch {
+                return null;
+            }
+        }
+        case "TOOL_DESCRIPTION": {
+            const domain = config.domains.find(d => d.tools.some(t => t.name === cause.tool));
+            const tool = domain?.tools.find(t => t.name === cause.tool);
+            if (!tool)
+                return null;
+            const before = tool.description;
+            const response = await cloud.completion({
+                tier: "balanced",
+                purpose: "eval",
+                messages: [
+                    { role: "system", content: "You improve tool descriptions for AI agents. Return ONLY the improved description text, nothing else." },
+                    { role: "user", content: `Current: "${before}"\nIssue: ${cause.detail}\nImprove it to be clearer about when to use this tool and expected inputs.` },
+                ],
+                temperature: 0,
+            });
+            tool.description = response.content.trim().replace(/^["']|["']$/g, "");
+            saveConfig(config, projectRoot);
+            return { iteration, timestamp, rootCause: cause.type, target: { type: "tool_description", tool: cause.tool }, before, after: tool.description, reason: cause.suggestedFix, triggeredBy: cause.scenario };
+        }
+        case "PROMPT_CLARITY": {
+            const agent = config.architecture.agents.find(a => a.name === cause.tool || a.domains.includes(cause.dimension));
+            if (!agent)
+                return null;
+            const before = agent.systemPrompt || "";
+            const response = await cloud.completion({
+                tier: "balanced",
+                purpose: "eval",
+                messages: [
+                    { role: "system", content: "You improve AI agent system prompts. Return ONLY the improved prompt text." },
+                    { role: "user", content: `Current prompt: "${before}"\nIssue: ${cause.detail}\nAdd specific instructions or few-shot examples to address this.` },
+                ],
+                temperature: 0,
+            });
+            agent.systemPrompt = response.content.trim();
+            saveConfig(config, projectRoot);
+            return { iteration, timestamp, rootCause: cause.type, target: { type: "system_prompt", agent: agent.name }, before, after: agent.systemPrompt, reason: cause.suggestedFix, triggeredBy: cause.scenario };
+        }
+        case "MODEL_CAPABILITY": {
+            const agent = config.architecture.agents.find(a => a.name === cause.tool || a.domains.some(d => cause.dimension.includes(d)));
+            if (!agent)
+                return null;
+            const before = agent.model;
+            const upgrades = { fast: "balanced", balanced: "powerful" };
+            const newModel = upgrades[agent.model];
+            if (!newModel)
+                return null;
+            agent.model = newModel;
+            saveConfig(config, projectRoot);
+            return { iteration, timestamp, rootCause: cause.type, target: { type: "model", agent: agent.name }, before, after: newModel, reason: cause.suggestedFix, triggeredBy: cause.scenario };
+        }
+        default:
+            return null;
+    }
+}
+function isAllowed(type, config) {
+    switch (type) {
+        case "SCHEMA_MISMATCH": return config.schemas;
+        case "TOOL_DESCRIPTION": return config.toolDescriptions;
+        case "PROMPT_CLARITY": return config.systemPrompts;
+        case "ROUTING_AMBIGUITY": return config.routerPrompt;
+        case "ARCHITECTURE": return config.architecture;
+        case "MODEL_CAPABILITY": return config.modelUpgrades;
+        default: return false;
+    }
+}
+function saveConfig(config, projectRoot) {
+    fs.writeFileSync(path.join(projectRoot, "sammy.config.json"), JSON.stringify(config, null, 2));
+}
+//# sourceMappingURL=refiner.js.map

package/dist/eval/refiner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"refiner.js","sourceRoot":"","sources":["../../src/eval/refiner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,MAAM,MAAM,aAAa,CAAC;AAOjC,MAAM,cAAc,GAAG;IACrB,iBAAiB;IACjB,kBAAkB;IAClB,gBAAgB;IAChB,mBAAmB;IACnB,cAAc;IACd,kBAAkB;CACV,CAAC;AAEX,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAA0B,EAC1B,MAAmB,EACnB,UAA4B,EAC5B,KAAuB,EACvB,KAAgB,EAChB,WAAmB;IAEnB,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,+BAA+B;IAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,IAAW,CAAC,CAAC;QACnD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,IAAW,CAAC,CAAC;QACnD,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,oCAAoC;QACpC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC;YAAE,SAAS;QAEjD,+BAA+B;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;aACxC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,EAAE,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;aACpF,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAE9B,IAAI,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAS;QAEnD,MAAM,oBAAoB,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACzD,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;aACxB,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;aAC5E,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,OAAO,CAC1C,CAAC,MAAM,CAAC;QAET,IAAI,oBAAoB,IAAI,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACjC,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;QACzF,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,yBAAyB,CAAC,CAAC;IACxF,MAAM,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACrG,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IAC1B,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAEjE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,KAAuC,EACvC,MAAmB,EACnB,KAAuB,EACvB,SAAiB,EACjB,WAAmB;IAEnB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,iBAAiB,CAAC,CAAC,CAAC;YACvB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAClF,MAAM,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAC;YAEvB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/C,yCAAyC;YACzC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC;gBACtC,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,2FAA2F,EAAE;oBACxH,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,uBAAuB,MAAM,YAAY,KAAK,CAAC,MAAM,iBAAiB,KAAK,CAAC,YAAY,0CAA0C,EAAE;iBAC9J;gBACD,WAAW,EAAE,CAAC;gBACd,cAAc,EAAE,MAAM;aACvB,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC9C,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC;gBAC3B,UAAU,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAChC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACzM,CAAC;YAAC,MAAM,CAAC;gBAAC,OAAO,IAAI,CAAC;YAAC,CAAC;QAC1B,CAAC;QAED,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACxB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAClF,MAAM,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAC;YAEvB,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC;gBACtC,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,uGAAuG,EAAE;oBACpI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,MAAM,aAAa,KAAK,CAAC,MAAM,6EAA6E,EAAE;iBACrJ;gBACD,WAAW,EAAE,CAAC;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;YACvE,UAAU,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAChC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC3M,CAAC;QAED,KAAK,gBAAgB,CAAC,CAAC,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;YACjH,IAAI,CAAC,KAAK;gBAAE,OAAO,IAAI,CAAC;YAExB,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC;gBACtC,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,4EAA4E,EAAE;oBACzG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,MAAM,aAAa,KAAK,CAAC,MAAM,mEAAmE,EAAE;iBAClJ;gBACD,WAAW,EAAE,CAAC;aACf,CAAC,CAAC;YAEH,KAAK,CAAC,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC7C,UAAU,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAChC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC3M,CAAC;QAED,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACxB,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9H,IAAI,CAAC,KAAK;gBAAE,OAAO,IAAI,CAAC;YAExB,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;YAC3B,MAAM,QAAQ,GAA2B,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;YACpF,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ;gBAAE,OAAO,IAAI,CAAC;YAE3B,KAAK,CAAC,KAAK,GAAG,QAAe,CAAC;YAC9B,UAAU,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAChC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QACzL,CAAC;QAED;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,MAAwB;IACvD,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,iBAAiB,CAAC,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC;QAC9C,KAAK,kBAAkB,CAAC,CAAC,OAAO,MAAM,CAAC,gBAAgB,CAAC;QACxD,KAAK,gBAAgB,CAAC,CAAC,OAAO,MAAM,CAAC,aAAa,CAAC;QACnD,KAAK,mBAAmB,CAAC,CAAC,OAAO,MAAM,CAAC,YAAY,CAAC;QACrD,KAAK,cAAc,CAAC,CAAC,OAAO,MAAM,CAAC,YAAY,CAAC;QAChD,KAAK,kBAAkB,CAAC,CAAC,OAAO,MAAM,CAAC,aAAa,CAAC;QACrD,OAAO,CAAC,CAAC,OAAO,KAAK,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAAmB,EAAE,WAAmB;IAC1D,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACjG,CAAC"}

package/dist/eval/runner.d.ts

@@ -0,0 +1,27 @@
+export interface EvalOptions {
+    maxIterations?: number;
+    dimensions?: string[];
+    agent?: string;
+    noRefine?: boolean;
+    ci?: boolean;
+    budget?: number;
+    generateOnly?: boolean;
+    fromIteration?: number;
+    dryRun?: boolean;
+}
+export interface EvalReport {
+    iterations: number;
+    passed: boolean;
+    terminationReason: string;
+    tokensUsed: number;
+    refinementsApplied: number;
+    escalations: number;
+    finalScores: Record<string, {
+        score: number;
+        passed: boolean;
+    }>;
+}
+export declare class EvalRunner {
+    run(options: EvalOptions): Promise<EvalReport>;
+}
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,WAAW;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACjE;AAED,qBAAa,UAAU;IACf,GAAG,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC;CA4NrD"}

package/dist/eval/runner.js

@@ -0,0 +1,198 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import chalk from "chalk";
+import ora from "ora";
+import { loadRuntimeConfig } from "../runtime/config-loader.js";
+import { ToolExecutor } from "../runtime/tool-executor.js";
+import { generateScenarios } from "./scenario-generator.js";
+import { scoreIteration } from "./scorer.js";
+import { diagnoseFailures } from "./diagnoser.js";
+import { applyRefinements } from "./refiner.js";
+import { checkGuards, updateLockedDimensions, checkRegression } from "./loop-guard.js";
+import { DEFAULT_EVAL_CONFIG, } from "./types.js";
+export class EvalRunner {
+    async run(options) {
+        const projectRoot = process.cwd();
+        console.log(chalk.bold("\n  Sammy Evaluation\n"));
+        // Load runtime config
+        const loadSpinner = ora("Loading configuration...").start();
+        let runtimeConfig;
+        try {
+            runtimeConfig = await loadRuntimeConfig({});
+            loadSpinner.succeed("Configuration loaded");
+        }
+        catch (err) {
+            loadSpinner.fail(`Failed to load config: ${err.message}`);
+            process.exit(1);
+        }
+        const config = runtimeConfig.config;
+        const evalConfig = {
+            ...DEFAULT_EVAL_CONFIG,
+            ...config.evaluation,
+        };
+        if (options.maxIterations)
+            evalConfig.guards.maxIterations = options.maxIterations;
+        if (options.budget)
+            evalConfig.guards.maxTokenBudget = options.budget;
+        // Build agent configs
+        const toolExecutor = new ToolExecutor(runtimeConfig.tools, {}, config.settings?.toolTimeout || 30000);
+        const agentConfigs = new Map();
+        for (const agent of config.architecture.agents) {
+            const agentTools = runtimeConfig.tools.filter(t => agent.domains.some(d => t.domain === d));
+            const domainDescs = config.domains
+                .filter(d => agent.domains.includes(d.name))
+                .map(d => `${d.name}: ${d.description}`)
+                .join("; ");
+            agentConfigs.set(agent.name, {
+                name: agent.name,
+                systemPrompt: agent.systemPrompt || `You specialize in: ${domainDescs}. Use the available tools.`,
+                tools: agentTools,
+                model: agent.model,
+                maxToolCalls: agent.maxToolCalls,
+            });
+        }
+        // Filter to specific agent if requested
+        if (options.agent) {
+            const kept = agentConfigs.get(options.agent);
+            agentConfigs.clear();
+            if (kept)
+                agentConfigs.set(options.agent, kept);
+        }
+        // Generate scenarios
+        const scenarioSpinner = ora("Generating test scenarios...").start();
+        const scenarios = await generateScenarios(config, evalConfig, runtimeConfig.cloudClient, projectRoot);
+        scenarioSpinner.succeed(`Generated ${scenarios.length} test scenarios`);
+        if (options.generateOnly) {
+            console.log(chalk.green(`\n  ✓ Scenarios saved to .sammy/eval/scenarios/generated.json\n`));
+            return {
+                iterations: 0, passed: false, terminationReason: "generate_only",
+                tokensUsed: 0, refinementsApplied: 0, escalations: 0, finalScores: {},
+            };
+        }
+        // Initialize loop state
+        const state = {
+            currentIteration: options.fromIteration || 1,
+            tokensUsed: 0,
+            scoreHistory: [],
+            lockedDimensions: new Set(),
+            appliedFixes: [],
+            skippedFixes: [],
+            escalations: [],
+        };
+        let lastScorecard = null;
+        // Main evaluation loop
+        while (true) {
+            console.log(chalk.bold(`\n  ITERATION ${state.currentIteration} / ${evalConfig.guards.maxIterations}`));
+            console.log("  " + "─".repeat(50));
+            // Score
+            const evalSpinner = ora(`Running ${scenarios.length} scenarios...`).start();
+            const { scorecard, tokensUsed } = await scoreIteration(scenarios, agentConfigs, toolExecutor, runtimeConfig.cloudClient, evalConfig, state.currentIteration, projectRoot);
+            state.tokensUsed += tokensUsed;
+            state.scoreHistory.push(scorecard);
+            lastScorecard = scorecard;
+            evalSpinner.succeed("Scenarios scored");
+            // Print scores
+            for (const [dim, score] of Object.entries(scorecard.dimensions)) {
+                const icon = score.passed ? chalk.green("✓") : chalk.red("✗");
+                const label = dim.padEnd(25);
+                const pct = dim === "responseQuality"
+                    ? `${score.score.toFixed(1)} / 10`
+                    : `${(score.score * 100).toFixed(1)}%`;
+                const extra = !score.passed ? chalk.dim(` [needs ≥${score.threshold}]`) : "";
+                console.log(`  ${icon} ${label} ${pct}${extra}`);
+            }
+            // Check if all pass
+            if (scorecard.overallPassed) {
+                state.terminationReason = "all_passed";
+                break;
+            }
+            // Check guards
+            const guard = checkGuards(state, evalConfig.guards);
+            if (!guard.shouldContinue) {
+                state.terminationReason = guard.reason;
+                break;
+            }
+            // No-refine mode
+            if (options.noRefine) {
+                state.terminationReason = "no_refine";
+                break;
+            }
+            // Diagnose
+            const diagSpinner = ora("Diagnosing failures...").start();
+            const diagnosis = await diagnoseFailures(scorecard, runtimeConfig.cloudClient, evalConfig, state.currentIteration, projectRoot);
+            diagSpinner.succeed(`Found ${diagnosis.rootCauses.length} root causes, ${diagnosis.escalations.length} escalations`);
+            state.escalations.push(...diagnosis.escalations);
+            // Print diagnosis summary
+            for (const cause of diagnosis.rootCauses.slice(0, 5)) {
+                console.log(chalk.dim(`  ├── ${cause.type} — ${cause.detail.slice(0, 60)}`));
+            }
+            // Refine
+            if (!options.dryRun) {
+                const refineSpinner = ora("Applying refinements...").start();
+                const actions = await applyRefinements(diagnosis, config, evalConfig.autoRefine, runtimeConfig.cloudClient, state, projectRoot);
+                refineSpinner.succeed(`Applied ${actions.length} refinements`);
+                for (const action of actions) {
+                    console.log(chalk.dim(`  ├── [${action.target.type}] ${action.reason.slice(0, 50)}`));
+                }
+            }
+            // Update locked dimensions
+            updateLockedDimensions(state, scorecard);
+            // Check regression
+            const regression = checkRegression(state, scorecard, evalConfig.guards);
+            if (regression.regressed) {
+                console.log(chalk.yellow(`  ⚠ Regression detected in ${regression.dimension}`));
+                if (state.appliedFixes.length > 0) {
+                    console.log(chalk.dim("  Rolling back last refinement..."));
+                    state.appliedFixes.pop();
+                }
+            }
+            state.currentIteration++;
+            // Final guard check
+            const postGuard = checkGuards(state, evalConfig.guards);
+            if (!postGuard.shouldContinue) {
+                state.terminationReason = postGuard.reason;
+                break;
+            }
+        }
+        // Final report
+        const passed = state.terminationReason === "all_passed";
+        const report = {
+            iterations: state.currentIteration,
+            passed,
+            terminationReason: state.terminationReason || "unknown",
+            tokensUsed: state.tokensUsed,
+            refinementsApplied: state.appliedFixes.length,
+            escalations: state.escalations.length,
+            finalScores: lastScorecard
+                ? Object.fromEntries(Object.entries(lastScorecard.dimensions).map(([k, v]) => [k, { score: v.score, passed: v.passed }]))
+                : {},
+        };
+        // Save report
+        const reportPath = path.join(projectRoot, ".sammy", "eval", "report.json");
+        fs.mkdirSync(path.dirname(reportPath), { recursive: true });
+        fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+        // Print summary
+        console.log(chalk.bold(`\n  ${"═".repeat(50)}`));
+        console.log(chalk.bold(`  RESULT: ${passed ? chalk.green("AGENTS READY ✓") : chalk.yellow("NEEDS ATTENTION ⚠")}`));
+        console.log();
+        console.log(chalk.dim(`  Terminated:    ${state.terminationReason} (${state.currentIteration} iterations)`));
+        console.log(chalk.dim(`  Refinements:   ${state.appliedFixes.length} applied`));
+        console.log(chalk.dim(`  Tokens used:   ${(state.tokensUsed / 1000).toFixed(0)}K`));
+        console.log(chalk.dim(`  Escalations:   ${state.escalations.length}`));
+        console.log(chalk.bold(`  ${"═".repeat(50)}\n`));
+        if (state.escalations.length > 0) {
+            console.log("  Manual fixes needed:");
+            for (const esc of state.escalations) {
+                console.log(chalk.dim(`  ├── ${esc.filePath}: ${esc.detail}`));
+                console.log(chalk.dim(`  │   Suggestion: ${esc.suggestion}`));
+            }
+            console.log();
+        }
+        // CI mode: exit with appropriate code
+        if (options.ci) {
+            process.exit(passed ? 0 : 1);
+        }
+        return report;
+    }
+}
+//# sourceMappingURL=runner.js.map

package/dist/eval/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvF,OAAO,EACL,mBAAmB,GAEpB,MAAM,YAAY,CAAC;AAwBpB,MAAM,OAAO,UAAU;IACrB,KAAK,CAAC,GAAG,CAAC,OAAoB;QAC5B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAElC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC;QAElD,sBAAsB;QACtB,MAAM,WAAW,GAAG,GAAG,CAAC,0BAA0B,CAAC,CAAC,KAAK,EAAE,CAAC;QAC5D,IAAI,aAAa,CAAC;QAClB,IAAI,CAAC;YACH,aAAa,GAAG,MAAM,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAC5C,WAAW,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,WAAW,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QACpC,MAAM,UAAU,GAAe;YAC7B,GAAG,mBAAmB;YACtB,GAAI,MAAc,CAAC,UAAU;SAC9B,CAAC;QAEF,IAAI,OAAO,CAAC,aAAa;YAAE,UAAU,CAAC,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QACnF,IAAI,OAAO,CAAC,MAAM;YAAE,UAAU,CAAC,MAAM,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;QAEtE,sBAAsB;QACtB,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,WAAW,IAAI,KAAK,CAAC,CAAC;QACtG,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;QAEvD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;YAC/C,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAChD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CACxC,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO;iBAC/B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;iBAC3C,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;iBACvC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEd,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE;gBAC3B,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,sBAAsB,WAAW,4BAA4B;gBACjG,KAAK,EAAE,UAAU;gBACjB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,YAAY,EAAE,KAAK,CAAC,YAAY;aACjC,CAAC,CAAC;QACL,CAAC;QAED,wCAAwC;QACxC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC7C,YAAY,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,IAAI;gBAAE,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAClD,CAAC;QAED,qBAAqB;QACrB,MAAM,eAAe,GAAG,GAAG,CAAC,8BAA8B,CAAC,CAAC,KAAK,EAAE,CAAC;QACpE,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACtG,eAAe,CAAC,OAAO,CAAC,aAAa,SAAS,CAAC,MAAM,iBAAiB,CAAC,CAAC;QAExE,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,iEAAiE,CAAC,CAAC,CAAC;YAC5F,OAAO;gBACL,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,eAAe;gBAChE,UAAU,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE;aACtE,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,MAAM,KAAK,GAAc;YACvB,gBAAgB,EAAE,OAAO,CAAC,aAAa,IAAI,CAAC;YAC5C,UAAU,EAAE,CAAC;YACb,YAAY,EAAE,EAAE;YAChB,gBAAgB,EAAE,IAAI,GAAG,EAAE;YAC3B,YAAY,EAAE,EAAE;YAChB,YAAY,EAAE,EAAE;YAChB,WAAW,EAAE,EAAE;SAChB,CAAC;QAEF,IAAI,aAAa,GAA+B,IAAI,CAAC;QAErD,uBAAuB;QACvB,OAAO,IAAI,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,gBAAgB,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;YACxG,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAEnC,QAAQ;YACR,MAAM,WAAW,GAAG,GAAG,CAAC,WAAW,SAAS,CAAC,MAAM,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC;YAC5E,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,MAAM,cAAc,CACpD,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,CAAC,WAAW,EAChE,UAAU,EAAE,KAAK,CAAC,gBAAgB,EAAE,WAAW,CAChD,CAAC;YACF,KAAK,CAAC,UAAU,IAAI,UAAU,CAAC;YAC/B,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACnC,aAAa,GAAG,SAAS,CAAC;YAC1B,WAAW,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAExC,eAAe;YACf,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9D,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC7B,MAAM,GAAG,GAAG,GAAG,KAAK,iBAAiB;oBACnC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;oBAClC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;gBACzC,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC;YACnD,CAAC;YAED,oBAAoB;YACpB,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;gBAC5B,KAAK,CAAC,iBAAiB,GAAG,YAAY,CAAC;gBACvC,MAAM;YACR,CAAC;YAED,eAAe;YACf,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACpD,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC1B,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,MAAM,CAAC;gBACvC,MAAM;YACR,CAAC;YAED,iBAAiB;YACjB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrB,KAAK,CAAC,iBAAiB,GAAG,WAAW,CAAC;gBACtC,MAAM;YACR,CAAC;YAED,WAAW;YACX,MAAM,WAAW,GAAG,GAAG,CAAC,wBAAwB,CAAC,CAAC,KAAK,EAAE,CAAC;YAC1D,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,UAAU,EAAE,KAAK,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;YAChI,WAAW,CAAC,OAAO,CAAC,SAAS,SAAS,CAAC,UAAU,CAAC,MAAM,iBAAiB,SAAS,CAAC,WAAW,CAAC,MAAM,cAAc,CAAC,CAAC;YAErH,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YAEjD,0BAA0B;YAC1B,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBACrD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,CAAC;YAED,SAAS;YACT,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,MAAM,aAAa,GAAG,GAAG,CAAC,yBAAyB,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC7D,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,UAAU,EACxC,aAAa,CAAC,WAAW,EAAE,KAAK,EAAE,WAAW,CAC9C,CAAC;gBACF,aAAa,CAAC,OAAO,CAAC,WAAW,OAAO,CAAC,MAAM,cAAc,CAAC,CAAC;gBAE/D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACxF,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,sBAAsB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAEzC,mBAAmB;YACnB,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACxE,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,8BAA8B,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBAChF,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC,CAAC;oBAC5D,KAAK,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;gBAC3B,CAAC;YACH,CAAC;YAED,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAEzB,oBAAoB;YACpB,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC9B,KAAK,CAAC,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC;gBAC3C,MAAM;YACR,CAAC;QACH,CAAC;QAED,eAAe;QACf,MAAM,MAAM,GAAG,KAAK,CAAC,iBAAiB,KAAK,YAAY,CAAC;QACxD,MAAM,MAAM,GAAe;YACzB,UAAU,EAAE,KAAK,CAAC,gBAAgB;YAClC,MAAM;YACN,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,IAAI,SAAS;YACvD,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,kBAAkB,EAAE,KAAK,CAAC,YAAY,CAAC,MAAM;YAC7C,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM;YACrC,WAAW,EAAE,aAAa;gBACxB,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACzH,CAAC,CAAC,EAAE;SACP,CAAC;QAEF,cAAc;QACd,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;QAC3E,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAE9D,gBAAgB;QAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC,CAAC;QACnH,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,KAAK,CAAC,iBAAiB,KAAK,KAAK,CAAC,gBAAgB,cAAc,CAAC,CAAC,CAAC;QAC7G,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,KAAK,CAAC,YAAY,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC;QAChF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAEjD,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YACtC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,sCAAsC;QACtC,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/eval/scenario-generator.d.ts

@@ -0,0 +1,5 @@
+import type { SammyConfig } from "../runtime/types.js";
+import type { SammyCloudClient } from "../cloud/sammy-cloud.js";
+import type { EvalScenario, EvalConfig } from "./types.js";
+export declare function generateScenarios(config: SammyConfig, evalConfig: EvalConfig, cloud: SammyCloudClient, projectRoot: string): Promise<EvalScenario[]>;
+//# sourceMappingURL=scenario-generator.d.ts.map

package/dist/eval/scenario-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario-generator.d.ts","sourceRoot":"","sources":["../../src/eval/scenario-generator.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAiB3D,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,gBAAgB,EACvB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,YAAY,EAAE,CAAC,CAiLzB"}