retestkit 1.4.1

package/src/tools/webtest/generate-tests.ts

@@ -0,0 +1,425 @@
+import { z } from "zod";
+import matter from "gray-matter";
+import type { McpTool, ToolResult } from "../../types/tool.js";
+import type { ServerContext } from "../../types/context.js";
+import type { WorkspaceManager } from "../../workspace/index.js";
+import type { SamplingClient } from "../../sampling/index.js";
+import type { ResourceManager } from "../../resources/index.js";
+import { buildTestGenerationPrompt } from "../../sampling/prompts.js";
+import { TestGenerationSchema, AnalysisIdSchema, type TestGeneration, type TestCase } from "./schemas.js";
+
+export const generateTestsInputSchema = z.object({
+  analysisId: AnalysisIdSchema,
+  appAnalysisUri: z
+    .string()
+    .optional()
+    .describe("URI of the app analysis resource. If not provided, uses the latest analysis."),
+  testStrategy: z
+    .enum(["comprehensive", "happy_path", "edge_cases", "critical_paths"])
+    .default("comprehensive")
+    .describe("Test generation strategy"),
+  focusFlows: z
+    .array(z.string())
+    .optional()
+    .describe("Specific flow IDs to focus test generation on"),
+  maxTests: z
+    .number()
+    .int()
+    .min(1)
+    .max(50)
+    .default(20)
+    .describe("Maximum number of test cases to generate"),
+  manualTests: z
+    .object({
+      tests: z.array(z.object({
+        id: z.string(),
+        name: z.string(),
+        purpose: z.string(),
+        category: z.enum(["happy_path", "edge_case", "error_handling", "boundary"]),
+        preconditions: z.array(z.string()),
+        steps: z.array(z.object({
+          stepNumber: z.number().int(),
+          action: z.string(),
+          target: z.string().optional(),
+          value: z.string().optional(),
+          expected: z.string().optional(),
+          element: z.string().optional(),
+          ref: z.string().optional(),
+        })),
+        expectedOutcomes: z.array(z.string()),
+        tags: z.array(z.string()).optional(),
+      })),
+      coverage: z.object({
+        flowsCovered: z.array(z.string()),
+        estimatedCoverage: z.string(),
+      }),
+    })
+    .optional()
+    .describe("Manual test generation result when sampling is unavailable. Provide this after executing the returned prompt."),
+});
+
+export type GenerateTestsInput = z.infer<typeof generateTestsInputSchema>;
+
+export function createGenerateTestsTool(
+  getContext: () => ServerContext & {
+    workspaceManager: WorkspaceManager;
+    samplingClient: SamplingClient;
+    resourceManager: ResourceManager;
+  }
+): McpTool<GenerateTestsInput> {
+  return {
+    name: "webtest_generate_tests",
+    description: `Generate test cases from application analysis.
+
+This tool creates structured test cases based on the app analysis:
+- Generates tests for identified user flows
+- Covers happy paths, edge cases, and error conditions
+- Creates step-by-step test instructions
+- Defines expected outcomes for each test
+
+Requires a completed app analysis. If sampling is unavailable, returns a prompt for manual execution.`,
+
+    inputSchema: generateTestsInputSchema,
+
+    async handler(input: GenerateTestsInput): Promise<ToolResult> {
+      const ctx = getContext();
+      const { logger, workspaceManager, samplingClient, resourceManager } = ctx;
+
+      const testLogger = logger.withCorrelation({
+        analysisId: input.analysisId,
+      });
+
+      testLogger.info("Starting test generation", {
+        strategy: input.testStrategy,
+        maxTests: input.maxTests,
+      });
+
+      try {
+        // Validate workspace exists
+        if (!(await workspaceManager.workspaceExists(input.analysisId))) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Error: Analysis workspace "${input.analysisId}" not found.`,
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        const workspace = await workspaceManager.readWorkspaceIndex(input.analysisId);
+
+        // Check for features
+        if (!workspace.features) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "Error: No features found. Run webtest_discover_features first.",
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        // Load features data
+        const featuresUri = input.appAnalysisUri || workspace.features.featuresUri;
+
+        let featuresContent: string;
+        let allFlows: Array<{ featureSlug: string; flows: unknown[] }> = [];
+
+        try {
+          const featuresData = await resourceManager.readResource(featuresUri);
+          featuresContent = featuresData.text || "";
+
+          // Load flows from all feature flow files
+          if (workspace.featureFlows) {
+            for (const flowRef of workspace.featureFlows) {
+              try {
+                const flowsContent = await resourceManager.readResource(flowRef.flowsUri);
+                const { data } = matter(flowsContent.text || "");
+                allFlows.push({
+                  featureSlug: flowRef.featureSlug,
+                  flows: (data as { flows?: unknown[] }).flows || [],
+                });
+              } catch {
+                // Flow file might not exist, skip
+              }
+            }
+          }
+        } catch (error) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Error loading features data: ${error instanceof Error ? error.message : "Unknown error"}`,
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        // Flatten flows for backward compatibility
+        const flowsData = {
+          flows: allFlows.flatMap(f => f.flows),
+          entities: workspace.features.features.flatMap(f => f.entities),
+        };
+
+        // Filter flows if specific ones requested
+        let targetFlows = flowsData.flows;
+        if (input.focusFlows && input.focusFlows.length > 0) {
+          targetFlows = flowsData.flows.filter((f: any) =>
+            input.focusFlows!.includes(f.id)
+          );
+        }
+
+        testLogger.info("Generating tests", {
+          flowCount: targetFlows.length,
+          strategy: input.testStrategy,
+        });
+
+        // Build strategy description
+        let strategyDescription: string;
+        switch (input.testStrategy) {
+          case "happy_path":
+            strategyDescription =
+              "Focus on successful user journeys. Test the main flows with valid inputs.";
+            break;
+          case "edge_cases":
+            strategyDescription =
+              "Focus on boundary conditions, unusual inputs, and corner cases.";
+            break;
+          case "critical_paths":
+            strategyDescription =
+              "Focus on business-critical functionality and high-risk areas.";
+            break;
+          case "comprehensive":
+          default:
+            strategyDescription =
+              "Generate a comprehensive test suite covering happy paths, edge cases, and error handling.";
+        }
+
+        // Determine test source: manual input, sampling, or fallback prompt
+        let testGeneration: TestGeneration;
+
+        if (input.manualTests) {
+          // Use manually provided tests (from fallback mode)
+          testLogger.info("Using manual tests input");
+          testGeneration = input.manualTests as TestGeneration;
+        } else if (!samplingClient.hasSampling()) {
+          // No sampling available and no manual input - return prompt for manual execution
+          const prompt = buildTestGenerationPrompt({
+            appAnalysis: featuresContent,
+            flows: JSON.stringify(targetFlows, null, 2),
+            strategy: strategyDescription,
+          });
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify(
+                  {
+                    needsManualInput: true,
+                    analysisId: input.analysisId,
+                    prompt,
+                    expectedResponseSchema: TestGenerationSchema._def,
+                    instructions:
+                      "Execute this prompt with your LLM, then call webtest_generate_tests again with the result in the 'manualTests' parameter",
+                  },
+                  null,
+                  2
+                ),
+              },
+            ],
+          };
+        } else {
+          // Request test generation via sampling
+          const samplingResult = await samplingClient.createMessage({
+            systemPrompt: `You are a QA engineer generating test cases for a web application.
+Generate ${input.maxTests} or fewer high-quality test cases based on the analysis.
+Each test case should be actionable and specific.`,
+            userPrompt: buildTestGenerationPrompt({
+              appAnalysis: featuresContent,
+              flows: JSON.stringify(targetFlows, null, 2),
+              strategy: strategyDescription,
+            }),
+            schema: TestGenerationSchema,
+            maxTokens: 8192,
+          });
+
+          if (!samplingResult.success || !samplingResult.data) {
+            testLogger.error("Test generation sampling failed", {
+              error: samplingResult.error,
+            });
+
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: `Error during test generation: ${samplingResult.error}`,
+                },
+              ],
+              isError: true,
+            };
+          }
+
+          testGeneration = samplingResult.data;
+        }
+
+        // Limit to maxTests
+        const tests = testGeneration.tests.slice(0, input.maxTests);
+
+        // Generate markdown report
+        const markdownReport = generateTestsMarkdown(tests, workspace, input.testStrategy);
+
+        // Create frontmatter with test data
+        const testsFrontmatter = {
+          tests,
+          coverage: testGeneration.coverage,
+          strategy: input.testStrategy,
+          generatedAt: new Date().toISOString(),
+        };
+
+        // Save tests as single markdown file with frontmatter
+        const { testsFilePath, testsUri, testCount } =
+          await workspaceManager.saveTests(input.analysisId, {
+            markdown: markdownReport,
+            frontmatter: testsFrontmatter,
+          });
+
+        await resourceManager.notifyListChanged();
+
+        // Categorize tests
+        const categories = {
+          happy_path: tests.filter((t) => t.category === "happy_path").length,
+          edge_case: tests.filter((t) => t.category === "edge_case").length,
+          error_handling: tests.filter((t) => t.category === "error_handling").length,
+          boundary: tests.filter((t) => t.category === "boundary").length,
+        };
+
+        const result = {
+          analysisId: input.analysisId,
+          testsFilePath,
+          testsUri,
+          testCount,
+          strategy: input.testStrategy,
+          categories,
+          coverage: testGeneration.coverage,
+          testSummary: tests.map((t) => ({
+            id: t.id,
+            name: t.name,
+            category: t.category,
+            stepCount: t.steps.length,
+          })),
+          nextSteps: [
+            `Use webtest_run_tests to execute individual tests`,
+            `Review generated tests at ${testsUri}`,
+            tests.length > 0
+              ? `First test to run: ${tests[0].id} - ${tests[0].name}`
+              : "",
+          ].filter(Boolean),
+        };
+
+        testLogger.info("Test generation completed", {
+          testCount,
+          categories,
+        });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(result, null, 2),
+            },
+          ],
+        };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : "Unknown error";
+        testLogger.error("Test generation failed", { error: message });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Error during test generation: ${message}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  };
+}
+
+function generateTestsMarkdown(
+  tests: TestCase[],
+  workspace: { url: string; domain: string },
+  strategy: string
+): string {
+  let md = `# Test Cases: ${workspace.domain}
+
+## Overview
+
+- **Target URL**: ${workspace.url}
+- **Strategy**: ${strategy}
+- **Total Tests**: ${tests.length}
+
+## Test Cases
+
+`;
+
+  for (const test of tests) {
+    md += `### ${test.id}: ${test.name}
+
+**Category**: ${test.category}
+**Purpose**: ${test.purpose}
+
+`;
+
+    if (test.preconditions.length > 0) {
+      md += `**Preconditions**:
+`;
+      for (const pre of test.preconditions) {
+        md += `- ${pre}\n`;
+      }
+      md += "\n";
+    }
+
+    md += `**Steps**:
+`;
+    for (const step of test.steps) {
+      let stepText = `${step.stepNumber}. ${step.action}`;
+      if (step.target) {
+        stepText += ` on "${step.target}"`;
+      }
+      if (step.value) {
+        stepText += ` with value "${step.value}"`;
+      }
+      md += stepText + "\n";
+      if (step.expected) {
+        md += `   - *Expected*: ${step.expected}\n`;
+      }
+    }
+
+    md += `
+**Expected Outcomes**:
+`;
+    for (const outcome of test.expectedOutcomes) {
+      md += `- ${outcome}\n`;
+    }
+
+    if (test.tags && test.tags.length > 0) {
+      md += `\n**Tags**: ${test.tags.join(", ")}\n`;
+    }
+
+    md += "\n---\n\n";
+  }
+
+  md += `
+*Generated by testing-mcp*
+`;
+
+  return md;
+}

package/src/tools/webtest/index.ts

@@ -0,0 +1,7 @@
+export * from "./schemas.js";
+export * from "./start-analysis.js";
+export * from "./crawl.js";
+export * from "./discover-features.js";
+export * from "./discover-flows.js";
+export * from "./generate-tests.js";
+export * from "./run-test-case.js";