webmcp-cli 1.0.0 → 1.2.1

package/dist/agent/features/simulation-judge.js

@@ -0,0 +1,276 @@
+/**
+ * Simulation Judge
+ *
+ * Uses LLM to evaluate simulation results and provide structured scoring
+ * with actionable improvement suggestions.
+ */
+import chalk from 'chalk';
+import { LLMClient, createLLMClient } from '../llm-client.js';
+import { parseJsonObject } from '../../llm/json-response.js';
+// ═══════════════════════════════════════════════════════
+// JUDGE PROMPTS
+// ═══════════════════════════════════════════════════════
+const JUDGE_SYSTEM_PROMPT = `You are an expert judge evaluating an AI agent's performance when interacting with website tools via the WebMCP protocol.
+
+You evaluate with strict criteria but fair judgment. Your evaluation should be:
+1. SPECIFIC — Point to exact moments in the conversation
+2. ACTIONABLE — Every criticism comes with a fix
+3. BALANCED — Acknowledge what worked well, not just failures
+4. QUANTIFIED — Use numbers, percentages, and scores`;
+function buildJudgmentPrompt(simulation, tools) {
+    const conversationLog = simulation.rounds.map(r => `[USER] ${r.userMessage}\n[AGENT] ${r.agentResponse.text || '(tool call)'}${r.agentResponse.toolCalls.length > 0
+        ? ` → called: ${r.agentResponse.toolCalls.map(tc => `${tc.toolName}(${JSON.stringify(tc.toolInput)})`).join(', ')}`
+        : ''}`).join('\n\n');
+    return `TOOLS AVAILABLE:
+${tools.map(t => `  ${t.name}: ${t.description}`).join('\n')}
+
+TEST CASE:
+  Prompt: "${simulation.testCase.prompt}"
+  Expected tool: ${simulation.testCase.expectedBehavior.toolName || 'none'}
+  Expected params: ${JSON.stringify(simulation.testCase.expectedBehavior.expectedParams)}
+  Category: ${simulation.testCase.category}
+  Difficulty: ${simulation.testCase.difficulty}/5
+
+CONVERSATION TRANSCRIPT:
+${conversationLog}
+
+FINAL OUTCOME:
+  Success: ${simulation.finalSuccess}
+  Tool invocations: ${simulation.toolInvocations}
+  Rounds used: ${simulation.totalRounds}
+${simulation.evaluation.failureReason ? `  Failure reason: ${simulation.evaluation.failureReason}` : ''}
+
+Judge this interaction on the following criteria:
+
+1. TOOL SELECTION ACCURACY (0-100)
+   Did the agent select the correct tool(s) for the user's request?
+   Did it avoid selecting wrong or unnecessary tools?
+
+2. PARAMETER ACCURACY (0-100)
+   Did the agent correctly extract and format parameters from the user's input?
+   Did it hallucinate any values not mentioned by the user?
+
+3. CONVERSATION EFFICIENCY (0-100)
+   Did the agent complete the task in the minimum number of exchanges?
+   Were there unnecessary clarifying questions or redundant tool calls?
+
+4. ERROR HANDLING (0-100)
+   When errors occurred, did the agent recover gracefully?
+   Did it provide helpful context to the user about what went wrong?
+
+5. USER EXPERIENCE (0-100)
+   Was the agent's communication clear and natural?
+   Did it keep the user informed about what it was doing?
+
+6. TOOL DEFINITION QUALITY (0-100)
+   Based on the agent's behavior, how well are the tools defined?
+   Did the tool descriptions/schemas cause any agent confusion?
+   This score reflects the TOOL QUALITY, not the agent's ability.
+
+OUTPUT FORMAT (strict JSON):
+{
+  "scores": {
+    "toolSelection": { "score": 85, "reasoning": "..." },
+    "parameterAccuracy": { "score": 72, "reasoning": "..." },
+    "conversationEfficiency": { "score": 60, "reasoning": "..." },
+    "errorHandling": { "score": 90, "reasoning": "..." },
+    "userExperience": { "score": 78, "reasoning": "..." },
+    "toolDefinitionQuality": { "score": 65, "reasoning": "..." }
+  },
+  "overallScore": 75,
+  "verdict": "PASS",
+  "keyMoments": [
+    {
+      "moment": "Round 1: Agent correctly identified search intent",
+      "impact": "positive",
+      "severity": "minor"
+    }
+  ],
+  "toolImprovements": [
+    {
+      "tool": "search-flights",
+      "issue": "Description doesn't mention airport code format",
+      "suggestedFix": "Add 'Use 3-letter IATA airport codes' to description",
+      "estimatedImpact": "Would improve parameter accuracy"
+    }
+  ],
+  "summary": "Overall assessment in 2-3 sentences"
+}`;
+}
+// ═══════════════════════════════════════════════════════
+// JUDGE FUNCTIONS
+// ═══════════════════════════════════════════════════════
+/**
+ * Judge a single simulation result
+ */
+export async function judgeSimulation(simulation, tools, llm) {
+    const client = llm || createLLMClient();
+    if (!client.isAvailable()) {
+        return createFallbackJudgment(simulation);
+    }
+    try {
+        const response = await client.chat([
+            { role: 'system', content: JUDGE_SYSTEM_PROMPT },
+            { role: 'user', content: buildJudgmentPrompt(simulation, tools) },
+        ]);
+        if (!response.text) {
+            throw new Error('No response from LLM');
+        }
+        return parseJudgmentResponse(response.text);
+    }
+    catch (error) {
+        console.warn(chalk.yellow(`Judgment failed: ${error instanceof Error ? error.message : error}`));
+        return createFallbackJudgment(simulation);
+    }
+}
+/**
+ * Judge multiple simulations and provide aggregate analysis
+ */
+export async function judgeSimulations(simulations, tools, llm) {
+    const client = llm || createLLMClient();
+    // Get individual judgments (sample if too many)
+    const sampleSize = Math.min(simulations.length, 10);
+    const sampled = simulations.slice(0, sampleSize);
+    const judgments = [];
+    for (const sim of sampled) {
+        const judgment = await judgeSimulation(sim, tools, client);
+        judgments.push(judgment);
+    }
+    // Aggregate results
+    const passCount = judgments.filter(j => j.verdict === 'PASS').length;
+    const partialCount = judgments.filter(j => j.verdict === 'PARTIAL').length;
+    const avgScore = judgments.reduce((sum, j) => sum + j.overallScore, 0) / judgments.length;
+    // Collect failure patterns
+    const failureMap = new Map();
+    for (const judgment of judgments) {
+        for (const improvement of judgment.toolImprovements) {
+            const key = improvement.issue;
+            if (!failureMap.has(key)) {
+                failureMap.set(key, { count: 0, tools: new Set() });
+            }
+            const entry = failureMap.get(key);
+            entry.count++;
+            entry.tools.add(improvement.tool);
+        }
+    }
+    const failurePatterns = Array.from(failureMap.entries())
+        .sort((a, b) => b[1].count - a[1].count)
+        .slice(0, 5)
+        .map(([pattern, data]) => {
+        const matchingJudgment = judgments.find(j => j.toolImprovements.some(i => i.issue === pattern));
+        const matchingImprovement = matchingJudgment?.toolImprovements.find(i => i.issue === pattern);
+        return {
+            pattern,
+            frequency: data.count,
+            affectedTools: Array.from(data.tools),
+            suggestedFix: matchingImprovement?.suggestedFix ?? 'Review tool definition',
+        };
+    });
+    // Collect unique improvements
+    const improvementMap = new Map();
+    for (const judgment of judgments) {
+        for (const improvement of judgment.toolImprovements) {
+            const key = `${improvement.tool}:${improvement.issue}`;
+            if (!improvementMap.has(key)) {
+                improvementMap.set(key, improvement);
+            }
+        }
+    }
+    const topImprovements = Array.from(improvementMap.values()).slice(0, 5);
+    return {
+        totalSimulations: simulations.length,
+        passRate: (passCount + partialCount * 0.5) / judgments.length,
+        avgScore,
+        failurePatterns,
+        topImprovements,
+        summary: `Evaluated ${simulations.length} simulations. ` +
+            `Pass rate: ${Math.round((passCount / judgments.length) * 100)}%. ` +
+            `Average score: ${Math.round(avgScore)}/100. ` +
+            `${failurePatterns.length > 0 && failurePatterns[0] ? `Top issue: ${failurePatterns[0].pattern}` : 'No major issues found.'}`,
+    };
+}
+/**
+ * Parse the LLM judgment response
+ */
+function parseJudgmentResponse(response) {
+    const parsed = parseJsonObject(response);
+    // Validate and fill defaults
+    return {
+        scores: parsed.scores || createDefaultScores(),
+        overallScore: parsed.overallScore || 50,
+        verdict: parsed.verdict || 'PARTIAL',
+        keyMoments: parsed.keyMoments || [],
+        toolImprovements: parsed.toolImprovements || [],
+        summary: parsed.summary || 'Evaluation complete.',
+    };
+}
+/**
+ * Create fallback judgment when LLM is unavailable
+ */
+function createFallbackJudgment(simulation) {
+    const eval_ = simulation.evaluation;
+    return {
+        scores: {
+            toolSelection: {
+                score: Math.round(eval_.routingAccuracy * 100),
+                reasoning: eval_.routingAccuracy === 1
+                    ? 'Correct tool selected on first try'
+                    : 'Tool selection issues detected',
+            },
+            parameterAccuracy: {
+                score: Math.round(eval_.parameterAccuracy * 100),
+                reasoning: eval_.parameterAccuracy >= 0.8
+                    ? 'Most parameters extracted correctly'
+                    : 'Some parameters were incorrect or missing',
+            },
+            conversationEfficiency: {
+                score: Math.round(eval_.roundEfficiency * 100),
+                reasoning: simulation.totalRounds === 1
+                    ? 'Completed in single round'
+                    : `Took ${simulation.totalRounds} rounds`,
+            },
+            errorHandling: {
+                score: simulation.finalSuccess ? 80 : 40,
+                reasoning: simulation.finalSuccess
+                    ? 'Task completed successfully'
+                    : 'Task did not complete successfully',
+            },
+            userExperience: {
+                score: 70,
+                reasoning: 'Heuristic evaluation - LLM not available for detailed analysis',
+            },
+            toolDefinitionQuality: {
+                score: Math.round((eval_.routingAccuracy + eval_.parameterAccuracy) * 50),
+                reasoning: 'Based on agent performance metrics',
+            },
+        },
+        overallScore: eval_.overallScore,
+        verdict: simulation.finalSuccess ? 'PASS' : eval_.overallScore >= 50 ? 'PARTIAL' : 'FAIL',
+        keyMoments: simulation.evaluation.failureReason ? [{
+                moment: simulation.evaluation.failureReason,
+                impact: 'negative',
+                severity: 'major',
+            }] : [],
+        toolImprovements: eval_.hallucinationRate > 0 ? [{
+                tool: simulation.testCase.expectedBehavior.toolName || 'unknown',
+                issue: 'Agent hallucinated parameters',
+                suggestedFix: 'Add clearer parameter descriptions to schema',
+                estimatedImpact: `Would reduce hallucination rate of ${Math.round(eval_.hallucinationRate * 100)}%`,
+            }] : [],
+        summary: simulation.finalSuccess
+            ? `Simulation passed. Tool ${simulation.testCase.expectedBehavior.toolName} invoked correctly.`
+            : `Simulation failed: ${simulation.evaluation.failureReason || 'Unknown reason'}`,
+    };
+}
+function createDefaultScores() {
+    return {
+        toolSelection: { score: 50, reasoning: 'Unable to evaluate' },
+        parameterAccuracy: { score: 50, reasoning: 'Unable to evaluate' },
+        conversationEfficiency: { score: 50, reasoning: 'Unable to evaluate' },
+        errorHandling: { score: 50, reasoning: 'Unable to evaluate' },
+        userExperience: { score: 50, reasoning: 'Unable to evaluate' },
+        toolDefinitionQuality: { score: 50, reasoning: 'Unable to evaluate' },
+    };
+}
+export default judgeSimulation;
+//# sourceMappingURL=simulation-judge.js.map

package/dist/agent/features/simulation-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"simulation-judge.js","sourceRoot":"","sources":["../../../src/agent/features/simulation-judge.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAG9D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAsD7D,0DAA0D;AAC1D,gBAAgB;AAChB,0DAA0D;AAE1D,MAAM,mBAAmB,GAAG;;;;;;qDAMyB,CAAC;AAEtD,SAAS,mBAAmB,CAC1B,UAA4B,EAC5B,KAAqB;IAErB,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAChD,UAAU,CAAC,CAAC,WAAW,aAAa,CAAC,CAAC,aAAa,CAAC,IAAI,IAAI,aAAa,GACvE,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;QAClC,CAAC,CAAC,cAAc,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAC/C,GAAG,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAClD,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAChB,CAAC,CAAC,EACN,EAAE,CACH,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEf,OAAO;EACP,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;aAG/C,UAAU,CAAC,QAAQ,CAAC,MAAM;mBACpB,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,IAAI,MAAM;qBACrD,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,cAAc,CAAC;cAC1E,UAAU,CAAC,QAAQ,CAAC,QAAQ;gBAC1B,UAAU,CAAC,QAAQ,CAAC,UAAU;;;EAG5C,eAAe;;;aAGJ,UAAU,CAAC,YAAY;sBACd,UAAU,CAAC,eAAe;iBAC/B,UAAU,CAAC,WAAW;EACrC,UAAU,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,qBAAqB,UAAU,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyDrG,CAAC;AACH,CAAC;AAED,0DAA0D;AAC1D,kBAAkB;AAClB,0DAA0D;AAE1D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAA4B,EAC5B,KAAqB,EACrB,GAAe;IAEf,MAAM,MAAM,GAAG,GAAG,IAAI,eAAe,EAAE,CAAC;IAExC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;QAC1B,OAAO,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;YACjC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,mBAAmB,EAAE;YAChD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE;SAClE,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACjG,OAAO,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,WAA+B,EAC/B,KAAqB,EACrB,GAAe;IAEf,MAAM,MAAM,GAAG,GAAG,IAAI,eAAe,EAAE,CAAC;IAExC,gDAAgD;IAChD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAEjD,MAAM,SAAS,GAAyB,EAAE,CAAC;IAE3C,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAC3D,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,oBAAoB;IACpB,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACrE,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAC3E,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;IAE1F,2BAA2B;IAC3B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAiD,CAAC;IAE5E,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,WAAW,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,WAAW,CAAC,KAAK,CAAC;YAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YACnC,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAqB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;SACvE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;SACvC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;QACvB,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAC1C,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAClD,CAAC;QACF,MAAM,mBAAmB,GAAG,gBAAgB,EAAE,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC;QAC9F,OAAO;YACL,OAAO;YACP,SAAS,EAAE,IAAI,CAAC,KAAK;YACrB,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YACrC,YAAY,EAAE,mBAAmB,EAAE,YAAY,IAAI,wBAAwB;SAC5E,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,8BAA8B;IAC9B,MAAM,cAAc,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC1D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,WAAW,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,GAAG,WAAW,CAAC,IAAI,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;YACvD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAExE,OAAO;QACL,gBAAgB,EAAE,WAAW,CAAC,MAAM;QACpC,QAAQ,EAAE,CAAC,SAAS,GAAG,YAAY,GAAG,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM;QAC7D,QAAQ;QACR,eAAe;QACf,eAAe;QACf,OAAO,EAAE,aAAa,WAAW,CAAC,MAAM,gBAAgB;YACtD,cAAc,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK;YACnE,kBAAkB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ;YAC9C,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,eAAe,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,wBAAwB,EAAE;KAChI,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,MAAM,MAAM,GAAG,eAAe,CAAqB,QAAQ,CAAC,CAAC;IAE7D,6BAA6B;IAC7B,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;QAC9C,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;QACvC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,SAAS;QACpC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE;QACnC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;QAC/C,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,sBAAsB;KAClD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,UAA4B;IAC1D,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC;IAEpC,OAAO;QACL,MAAM,EAAE;YACN,aAAa,EAAE;gBACb,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC9C,SAAS,EAAE,KAAK,CAAC,eAAe,KAAK,CAAC;oBACpC,CAAC,CAAC,oCAAoC;oBACtC,CAAC,CAAC,gCAAgC;aACrC;YACD,iBAAiB,EAAE;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,iBAAiB,GAAG,GAAG,CAAC;gBAChD,SAAS,EAAE,KAAK,CAAC,iBAAiB,IAAI,GAAG;oBACvC,CAAC,CAAC,qCAAqC;oBACvC,CAAC,CAAC,2CAA2C;aAChD;YACD,sBAAsB,EAAE;gBACtB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC9C,SAAS,EAAE,UAAU,CAAC,WAAW,KAAK,CAAC;oBACrC,CAAC,CAAC,2BAA2B;oBAC7B,CAAC,CAAC,QAAQ,UAAU,CAAC,WAAW,SAAS;aAC5C;YACD,aAAa,EAAE;gBACb,KAAK,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;gBACxC,SAAS,EAAE,UAAU,CAAC,YAAY;oBAChC,CAAC,CAAC,6BAA6B;oBAC/B,CAAC,CAAC,oCAAoC;aACzC;YACD,cAAc,EAAE;gBACd,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,gEAAgE;aAC5E;YACD,qBAAqB,EAAE;gBACrB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;gBACzE,SAAS,EAAE,oCAAoC;aAChD;SACF;QACD,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,OAAO,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;QACzF,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;gBACjD,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,aAAa;gBAC3C,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,OAAO;aAClB,CAAC,CAAC,CAAC,CAAC,EAAE;QACP,gBAAgB,EAAE,KAAK,CAAC,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,IAAI,SAAS;gBAChE,KAAK,EAAE,+BAA+B;gBACtC,YAAY,EAAE,8CAA8C;gBAC5D,eAAe,EAAE,sCAAsC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,iBAAiB,GAAG,GAAG,CAAC,GAAG;aACpG,CAAC,CAAC,CAAC,CAAC,EAAE;QACP,OAAO,EAAE,UAAU,CAAC,YAAY;YAC9B,CAAC,CAAC,2BAA2B,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,qBAAqB;YAC/F,CAAC,CAAC,sBAAsB,UAAU,CAAC,UAAU,CAAC,aAAa,IAAI,gBAAgB,EAAE;KACpF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,aAAa,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,oBAAoB,EAAE;QAC7D,iBAAiB,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,oBAAoB,EAAE;QACjE,sBAAsB,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,oBAAoB,EAAE;QACtE,aAAa,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,oBAAoB,EAAE;QAC7D,cAAc,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,oBAAoB,EAAE;QAC9D,qBAAqB,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,oBAAoB,EAAE;KACtE,CAAC;AACJ,CAAC;AAED,eAAe,eAAe,CAAC"}

package/dist/agent/features/test-case-generator.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Test Case Generator
+ *
+ * Uses LLM to generate realistic user prompts for testing WebMCP tools.
+ * Generates diverse test cases covering happy paths, edge cases, and adversarial inputs.
+ */
+import { LLMClient } from '../llm-client.js';
+import type { DetectedTool } from '../../core/types/tool.js';
+export type TestCaseCategory = 'happy_path' | 'partial_info' | 'casual' | 'verbose' | 'ambiguous' | 'adversarial' | 'out_of_scope';
+export interface TestCase {
+    id: string;
+    prompt: string;
+    category: TestCaseCategory;
+    difficulty: 1 | 2 | 3 | 4 | 5;
+    expectedBehavior: {
+        shouldInvokeTool: boolean;
+        toolName: string | null;
+        expectedParams: Record<string, unknown>;
+        paramsExplanation: Record<string, string>;
+        missingParams: string[];
+        shouldAskForMissing: boolean;
+        acceptableAlternatives: string[];
+    };
+    failureModes: string[];
+}
+export interface TestCaseGenerationConfig {
+    count: number;
+    categories: TestCaseCategory[];
+    model?: string;
+}
+/**
+ * Generate test cases for the given tools
+ */
+export declare function generateTestCases(tools: DetectedTool[], siteContext: string, config: TestCaseGenerationConfig, llm?: LLMClient): Promise<TestCase[]>;
+export default generateTestCases;

package/dist/agent/features/test-case-generator.js

@@ -0,0 +1,257 @@
+/**
+ * Test Case Generator
+ *
+ * Uses LLM to generate realistic user prompts for testing WebMCP tools.
+ * Generates diverse test cases covering happy paths, edge cases, and adversarial inputs.
+ */
+import chalk from 'chalk';
+import { LLMClient, createLLMClient } from '../llm-client.js';
+import { parseJsonObject } from '../../llm/json-response.js';
+// ═══════════════════════════════════════════════════════
+// PROMPTS
+// ═══════════════════════════════════════════════════════
+const SYSTEM_PROMPT = `You are a QA engineer specialized in testing AI agent tool interactions. Your job is to generate realistic test cases — natural language prompts that real users would say to an AI assistant, expecting specific tools to be invoked.
+
+IMPORTANT RULES:
+1. Generate diverse phrasings — formal, casual, abbreviated, verbose
+2. Include edge cases — partial information, ambiguous requests
+3. Include realistic mistakes — wrong terminology, vague descriptions
+4. For each prompt, specify the EXPECTED correct behavior
+5. Be creative — real users are unpredictable
+6. Output valid JSON only — no markdown, no commentary outside the JSON`;
+function buildGenerationPrompt(tools, siteContext, config) {
+    const { count, categories } = config;
+    return `Given the following WebMCP tools registered on a ${siteContext} website, generate exactly ${count} test case prompts.
+
+TOOLS AVAILABLE:
+${tools.map(t => `
+Tool: "${t.name}"
+Description: "${t.description}"
+Parameters: ${JSON.stringify(t.inputSchema, null, 2)}
+`).join('\n---\n')}
+
+Generate test cases across these categories: ${categories.join(', ')}
+
+Category definitions:
+- "happy_path": Clear, complete requests that should invoke the tool correctly with all required parameters
+- "partial_info": Requests missing some required info — agent should ask for clarification OR use defaults
+- "casual": Very informal, shorthand, or slang phrasings ("yo find me flights to nyc asap")
+- "verbose": Overly detailed requests with extra context the agent must filter
+- "ambiguous": Requests that COULD match multiple tools — tests disambiguation
+- "adversarial": Typos, wrong terminology, misleading phrasing, or requests just outside the tool's scope
+- "out_of_scope": Requests that NONE of the available tools should handle — agent should decline
+
+For each test case, provide:
+- The natural language prompt a user would say
+- Which tool(s) should be invoked (or "none" if out of scope)
+- The expected parameters the agent should extract
+- What a correct agent response looks like
+- What category this test case belongs to
+- Difficulty rating for the agent (1=trivial, 5=very hard)
+
+OUTPUT FORMAT (strict JSON):
+{
+  "testCases": [
+    {
+      "id": "TC-001",
+      "prompt": "the exact user prompt",
+      "category": "happy_path",
+      "difficulty": 2,
+      "expectedBehavior": {
+        "shouldInvokeTool": true,
+        "toolName": "tool-name",
+        "expectedParams": {
+          "param1": "value1",
+          "param2": "value2"
+        },
+        "paramsExplanation": {
+          "param1": "Explicitly stated as X",
+          "param2": "Inferred from Y"
+        },
+        "missingParams": ["param3"],
+        "shouldAskForMissing": true,
+        "acceptableAlternatives": [
+          "Agent could interpret 'X' as 'Y'"
+        ]
+      },
+      "failureModes": [
+        "Agent might not recognize X as Y",
+        "Agent might pick wrong tool"
+      ]
+    }
+  ]
+}`;
+}
+// ═══════════════════════════════════════════════════════
+// GENERATOR
+// ═══════════════════════════════════════════════════════
+/**
+ * Generate test cases for the given tools
+ */
+export async function generateTestCases(tools, siteContext, config, llm) {
+    const client = llm || createLLMClient({ model: config.model });
+    if (!client.isAvailable()) {
+        // Return fallback test cases when no API key
+        return generateFallbackTestCases(tools, config);
+    }
+    try {
+        const response = await client.chat([
+            { role: 'system', content: SYSTEM_PROMPT },
+            { role: 'user', content: buildGenerationPrompt(tools, siteContext, config) },
+        ]);
+        if (!response.text) {
+            throw new Error('No response from LLM');
+        }
+        // Extract JSON from response
+        const parsed = parseTestCaseResponse(response.text);
+        return parsed.testCases;
+    }
+    catch (error) {
+        console.warn(chalk.yellow(`Test case generation failed: ${error instanceof Error ? error.message : error}`));
+        return generateFallbackTestCases(tools, config);
+    }
+}
+/**
+ * Parse the LLM response, handling code blocks
+ */
+function parseTestCaseResponse(response) {
+    return parseJsonObject(response);
+}
+/**
+ * Generate basic fallback test cases when LLM is unavailable
+ */
+function generateFallbackTestCases(tools, config) {
+    const testCases = [];
+    let id = 1;
+    for (const tool of tools) {
+        // Extract parameters from schema
+        const properties = tool.inputSchema?.properties || {};
+        const required = (tool.inputSchema?.required || []);
+        const paramNames = Object.keys(properties);
+        // Happy path test
+        if (config.categories.includes('happy_path')) {
+            const params = {};
+            const explanations = {};
+            for (const param of paramNames) {
+                const prop = properties[param];
+                params[param] = getExampleValue(param, prop);
+                explanations[param] = 'Example value for testing';
+            }
+            testCases.push({
+                id: `TC-${String(id++).padStart(3, '0')}`,
+                prompt: `Use ${tool.name} with ${paramNames.join(', ')}`,
+                category: 'happy_path',
+                difficulty: 1,
+                expectedBehavior: {
+                    shouldInvokeTool: true,
+                    toolName: tool.name,
+                    expectedParams: params,
+                    paramsExplanation: explanations,
+                    missingParams: [],
+                    shouldAskForMissing: false,
+                    acceptableAlternatives: [],
+                },
+                failureModes: ['Agent might not recognize the tool name'],
+            });
+        }
+        // Partial info test
+        if (config.categories.includes('partial_info') && required.length > 0) {
+            const params = {};
+            const explanations = {};
+            const firstRequired = required[0];
+            if (firstRequired) {
+                const prop = properties[firstRequired];
+                params[firstRequired] = getExampleValue(firstRequired, prop);
+                explanations[firstRequired] = 'Only provided this parameter';
+            }
+            testCases.push({
+                id: `TC-${String(id++).padStart(3, '0')}`,
+                prompt: `I want to ${tool.name.replace(/[-_]/g, ' ')}`,
+                category: 'partial_info',
+                difficulty: 3,
+                expectedBehavior: {
+                    shouldInvokeTool: true,
+                    toolName: tool.name,
+                    expectedParams: params,
+                    paramsExplanation: explanations,
+                    missingParams: required.slice(1),
+                    shouldAskForMissing: true,
+                    acceptableAlternatives: [],
+                },
+                failureModes: ['Agent might guess missing values'],
+            });
+        }
+        // Casual test
+        if (config.categories.includes('casual')) {
+            testCases.push({
+                id: `TC-${String(id++).padStart(3, '0')}`,
+                prompt: `hey can u ${tool.name.replace(/[-_]/g, ' ')} for me`,
+                category: 'casual',
+                difficulty: 2,
+                expectedBehavior: {
+                    shouldInvokeTool: true,
+                    toolName: tool.name,
+                    expectedParams: {},
+                    paramsExplanation: {},
+                    missingParams: required,
+                    shouldAskForMissing: true,
+                    acceptableAlternatives: [],
+                },
+                failureModes: ['Agent might not understand casual phrasing'],
+            });
+        }
+    }
+    // Out of scope test
+    if (config.categories.includes('out_of_scope')) {
+        testCases.push({
+            id: `TC-${String(id++).padStart(3, '0')}`,
+            prompt: 'What is the meaning of life?',
+            category: 'out_of_scope',
+            difficulty: 1,
+            expectedBehavior: {
+                shouldInvokeTool: false,
+                toolName: null,
+                expectedParams: {},
+                paramsExplanation: {},
+                missingParams: [],
+                shouldAskForMissing: false,
+                acceptableAlternatives: [],
+            },
+            failureModes: ['Agent might try to use a tool anyway'],
+        });
+    }
+    return testCases.slice(0, config.count);
+}
+/**
+ * Get an example value for a parameter
+ */
+function getExampleValue(name, prop) {
+    const type = prop['type'];
+    const enumValues = prop['enum'];
+    if (enumValues && enumValues.length > 0) {
+        return enumValues[0];
+    }
+    switch (type) {
+        case 'string':
+            if (name.toLowerCase().includes('date'))
+                return '2026-06-15';
+            if (name.toLowerCase().includes('email'))
+                return 'test@example.com';
+            if (name.toLowerCase().includes('url'))
+                return 'https://example.com';
+            return 'example_value';
+        case 'number':
+        case 'integer':
+            return 1;
+        case 'boolean':
+            return true;
+        case 'array':
+            return [];
+        case 'object':
+            return {};
+        default:
+            return 'example';
+    }
+}
+export default generateTestCases;
+//# sourceMappingURL=test-case-generator.js.map

package/dist/agent/features/test-case-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-case-generator.js","sourceRoot":"","sources":["../../../src/agent/features/test-case-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAsC7D,0DAA0D;AAC1D,UAAU;AACV,0DAA0D;AAE1D,MAAM,aAAa,GAAG;;;;;;;;wEAQkD,CAAC;AAEzE,SAAS,qBAAqB,CAC5B,KAAqB,EACrB,WAAmB,EACnB,MAAgC;IAEhC,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAErC,OAAO,oDAAoD,WAAW,8BAA8B,KAAK;;;EAGzG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC,IAAI;gBACC,CAAC,CAAC,WAAW;cACf,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;CACnD,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;;+CAE6B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkDlE,CAAC;AACH,CAAC;AAED,0DAA0D;AAC1D,YAAY;AACZ,0DAA0D;AAE1D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAqB,EACrB,WAAmB,EACnB,MAAgC,EAChC,GAAe;IAEf,MAAM,MAAM,GAAG,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAE/D,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;QAC1B,6CAA6C;QAC7C,OAAO,yBAAyB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;YACjC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE;SAC7E,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,6BAA6B;QAC7B,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpD,OAAO,MAAM,CAAC,SAAS,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gCAAgC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC7G,OAAO,yBAAyB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,OAAO,eAAe,CAA4B,QAAQ,CAAC,CAAC;AAC9D,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAChC,KAAqB,EACrB,MAAgC;IAEhC,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,IAAI,EAAE,GAAG,CAAC,CAAC;IAEX,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,iCAAiC;QACjC,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,UAAU,IAAI,EAAE,CAAC;QACtD,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAa,CAAC;QAChE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAE3C,kBAAkB;QAClB,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7C,MAAM,MAAM,GAA4B,EAAE,CAAC;YAC3C,MAAM,YAAY,GAA2B,EAAE,CAAC;YAEhD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAA4B,CAAC;gBAC1D,MAAM,CAAC,KAAK,CAAC,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC7C,YAAY,CAAC,KAAK,CAAC,GAAG,2BAA2B,CAAC;YACpD,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,MAAM,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBACzC,MAAM,EAAE,OAAO,IAAI,CAAC,IAAI,SAAS,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACxD,QAAQ,EAAE,YAAY;gBACtB,UAAU,EAAE,CAAC;gBACb,gBAAgB,EAAE;oBAChB,gBAAgB,EAAE,IAAI;oBACtB,QAAQ,EAAE,IAAI,CAAC,IAAI;oBACnB,cAAc,EAAE,MAAM;oBACtB,iBAAiB,EAAE,YAAY;oBAC/B,aAAa,EAAE,EAAE;oBACjB,mBAAmB,EAAE,KAAK;oBAC1B,sBAAsB,EAAE,EAAE;iBAC3B;gBACD,YAAY,EAAE,CAAC,yCAAyC,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QAED,oBAAoB;QACpB,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,MAAM,GAA4B,EAAE,CAAC;YAC3C,MAAM,YAAY,GAA2B,EAAE,CAAC;YAChD,MAAM,aAAa,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,IAAI,GAAG,UAAU,CAAC,aAAa,CAA4B,CAAC;gBAClE,MAAM,CAAC,aAAa,CAAC,GAAG,eAAe,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;gBAC7D,YAAY,CAAC,aAAa,CAAC,GAAG,8BAA8B,CAAC;YAC/D,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,MAAM,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBACzC,MAAM,EAAE,aAAa,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE;gBACtD,QAAQ,EAAE,cAAc;gBACxB,UAAU,EAAE,CAAC;gBACb,gBAAgB,EAAE;oBAChB,gBAAgB,EAAE,IAAI;oBACtB,QAAQ,EAAE,IAAI,CAAC,IAAI;oBACnB,cAAc,EAAE,MAAM;oBACtB,iBAAiB,EAAE,YAAY;oBAC/B,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;oBAChC,mBAAmB,EAAE,IAAI;oBACzB,sBAAsB,EAAE,EAAE;iBAC3B;gBACD,YAAY,EAAE,CAAC,kCAAkC,CAAC;aACnD,CAAC,CAAC;QACL,CAAC;QAED,cAAc;QACd,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,MAAM,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBACzC,MAAM,EAAE,aAAa,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS;gBAC7D,QAAQ,EAAE,QAAQ;gBAClB,UAAU,EAAE,CAAC;gBACb,gBAAgB,EAAE;oBAChB,gBAAgB,EAAE,IAAI;oBACtB,QAAQ,EAAE,IAAI,CAAC,IAAI;oBACnB,cAAc,EAAE,EAAE;oBAClB,iBAAiB,EAAE,EAAE;oBACrB,aAAa,EAAE,QAAQ;oBACvB,mBAAmB,EAAE,IAAI;oBACzB,sBAAsB,EAAE,EAAE;iBAC3B;gBACD,YAAY,EAAE,CAAC,4CAA4C,CAAC;aAC7D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QAC/C,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,MAAM,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YACzC,MAAM,EAAE,8BAA8B;YACtC,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,CAAC;YACb,gBAAgB,EAAE;gBAChB,gBAAgB,EAAE,KAAK;gBACvB,QAAQ,EAAE,IAAI;gBACd,cAAc,EAAE,EAAE;gBAClB,iBAAiB,EAAE,EAAE;gBACrB,aAAa,EAAE,EAAE;gBACjB,mBAAmB,EAAE,KAAK;gBAC1B,sBAAsB,EAAE,EAAE;aAC3B;YACD,YAAY,EAAE,CAAC,sCAAsC,CAAC;SACvD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,IAA6B;IAClE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAW,CAAC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAc,CAAC;IAE7C,IAAI,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ;YACX,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,YAAY,CAAC;YAC7D,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,kBAAkB,CAAC;YACpE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,OAAO,qBAAqB,CAAC;YACrE,OAAO,eAAe,CAAC;QACzB,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC;QACX,KAAK,SAAS;YACZ,OAAO,IAAI,CAAC;QACd,KAAK,OAAO;YACV,OAAO,EAAE,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,EAAE,CAAC;QACZ;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,eAAe,iBAAiB,CAAC"}

package/dist/agent/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Agent Module
+ *
+ * LLM-powered agent simulation for testing WebMCP tools.
+ */
+export { LLMClient, createLLMClient, type LLMClientConfig, type LLMMessage, type LLMTool, type LLMToolCall, type LLMResponse, } from './llm-client.js';
+export { generateTestCases, runSimulation, runSimulations, judgeSimulation, judgeSimulations, type TestCase, type TestCaseCategory, type TestCaseGenerationConfig, type SimulationConfig, type SimulationResult, type SimulationRound, type ToolResult, type RoundEvaluation, type SimulationJudgment, type JudgmentResult, type KeyMoment, type ToolImprovement, type AggregateJudgment, type FailurePattern, } from './features/index.js';

package/dist/agent/index.js

@@ -0,0 +1,10 @@
+/**
+ * Agent Module
+ *
+ * LLM-powered agent simulation for testing WebMCP tools.
+ */
+// LLM Client
+export { LLMClient, createLLMClient, } from './llm-client.js';
+// Features
+export { generateTestCases, runSimulation, runSimulations, judgeSimulation, judgeSimulations, } from './features/index.js';
+//# sourceMappingURL=index.js.map

package/dist/agent/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/agent/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,aAAa;AACb,OAAO,EACL,SAAS,EACT,eAAe,GAMhB,MAAM,iBAAiB,CAAC;AAEzB,WAAW;AACX,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,eAAe,EACf,gBAAgB,GAejB,MAAM,qBAAqB,CAAC"}