retestkit 1.4.1 → 1.5.0

package/src/tools/webtest/run-test-case.ts

@@ -1,508 +0,0 @@
-import { z } from "zod";
-import { join } from "node:path";
-import matter from "gray-matter";
-import type { McpTool, ToolResult } from "../../types/tool.js";
-import type { ServerContext } from "../../types/context.js";
-import type { WorkspaceManager, TestCase, TestStepResult } from "../../workspace/index.js";
-import type { PlaywrightClient } from "../../playwright-client/index.js";
-import type { SamplingClient } from "../../sampling/index.js";
-import type { CancellationRegistry, ProgressEmitter } from "../../progress/index.js";
-import type { ResourceManager } from "../../resources/index.js";
-import { CancellationError } from "../../progress/index.js";
-import { buildTestEvaluationPrompt } from "../../sampling/prompts.js";
-import { TestEvaluationSchema, AnalysisIdSchema } from "./schemas.js";
-
-export const runTestCaseInputSchema = z.object({
-  analysisId: AnalysisIdSchema,
-  testCaseId: z.string().describe("ID of the test case to run"),
-  testsUri: z
-    .string()
-    .optional()
-    .describe("URI of the tests resource. If not provided, uses the latest tests."),
-  runOptions: z
-    .object({
-      captureEvidence: z.boolean().default(true),
-      stopOnFailure: z.boolean().default(true),
-      retryFailedSteps: z.boolean().default(false),
-    })
-    .optional()
-    .describe("Test run options"),
-});
-
-export type RunTestCaseInput = z.infer<typeof runTestCaseInputSchema>;
-
-export function createRunTestCaseTool(
-  getContext: () => ServerContext & {
-    workspaceManager: WorkspaceManager;
-    playwrightClient: PlaywrightClient;
-    samplingClient: SamplingClient;
-    cancellationRegistry: CancellationRegistry;
-    progressEmitter: ProgressEmitter;
-    resourceManager: ResourceManager;
-  }
-): McpTool<RunTestCaseInput> {
-  return {
-    name: "webtest_run_test",
-    description: `Execute a single test case against the web application.
-
-This tool runs one generated test case:
-- Executes each step using Playwright browser automation
-- Captures evidence (screenshots, snapshots) at each step
-- Evaluates pass/fail using AI-powered assertion checking
-- Reports detailed results with evidence links
-
-Requires generated test cases. Progress is reported throughout execution.`,
-
-    inputSchema: runTestCaseInputSchema,
-
-    async handler(input: RunTestCaseInput): Promise<ToolResult> {
-      const ctx = getContext();
-      const {
-        logger,
-        workspaceManager,
-        playwrightClient,
-        samplingClient,
-        cancellationRegistry,
-        progressEmitter,
-        resourceManager,
-      } = ctx;
-
-      const requestId = `test-${input.analysisId}-${input.testCaseId}-${Date.now()}`;
-      const runLogger = logger.withCorrelation({
-        analysisId: input.analysisId,
-        requestId,
-      });
-
-      runLogger.info("Starting test case execution", {
-        testCaseId: input.testCaseId,
-      });
-
-      // Register for cancellation
-      cancellationRegistry.register(requestId);
-
-      try {
-        // Validate workspace exists
-        if (!(await workspaceManager.workspaceExists(input.analysisId))) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Error: Analysis workspace "${input.analysisId}" not found.`,
-              },
-            ],
-            isError: true,
-          };
-        }
-
-        const workspace = await workspaceManager.readWorkspaceIndex(input.analysisId);
-
-        // Check for tests
-        if (!workspace.tests) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: "Error: No tests found. Run webtest_generate_tests first.",
-              },
-            ],
-            isError: true,
-          };
-        }
-
-        // Load tests
-        const testsUri = input.testsUri || workspace.tests.testsUri;
-
-        let testsData: { tests: TestCase[] };
-        try {
-          const testsContent = await resourceManager.readResource(testsUri);
-          // Parse tests from markdown frontmatter
-          const { data } = matter(testsContent.text || "");
-          testsData = data as { tests: TestCase[] };
-        } catch (error) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Error loading tests: ${error instanceof Error ? error.message : "Unknown error"}`,
-              },
-            ],
-            isError: true,
-          };
-        }
-
-        // Find test case
-        const testCase = testsData.tests.find((t) => t.id === input.testCaseId);
-        if (!testCase) {
-          return {
-            content: [
-              {
-                type: "text",
-                text: `Error: Test case "${input.testCaseId}" not found. Available tests: ${testsData.tests.map((t) => t.id).join(", ")}`,
-              },
-            ],
-            isError: true,
-          };
-        }
-
-        const runOptions = input.runOptions ?? {
-          captureEvidence: true,
-          stopOnFailure: true,
-          retryFailedSteps: false,
-        };
-
-        // Create test run
-        const { runId, runPath } = await workspaceManager.createTestRun(
-          input.analysisId,
-          { testCaseId: input.testCaseId, testName: testCase.name }
-        );
-
-        const stepLogger = runLogger.withCorrelation({ testRunId: runId });
-        stepLogger.info("Test run created", { runId, testName: testCase.name });
-
-        await resourceManager.notifyListChanged();
-
-        // Ensure Playwright is connected
-        if (!playwrightClient.isConnected()) {
-          await playwrightClient.connect();
-        }
-
-        // Navigate to start URL
-        await playwrightClient.navigate(workspace.url);
-
-        // Execute test steps
-        const stepResults: TestStepResult[] = [];
-        let overallStatus: "passed" | "failed" | "error" = "passed";
-        let failureStep: number | undefined;
-
-        for (const step of testCase.steps) {
-          // Check cancellation
-          cancellationRegistry.checkCancelled(requestId);
-
-          // Emit progress
-          progressEmitter.emit({
-            progressToken: requestId,
-            progress: step.stepNumber,
-            total: testCase.steps.length,
-            message: `Step ${step.stepNumber}/${testCase.steps.length}: ${step.action}`,
-          });
-
-          stepLogger.info("Executing test step", {
-            stepNumber: step.stepNumber,
-            action: step.action,
-          });
-
-          const stepResult: TestStepResult = {
-            stepNumber: step.stepNumber,
-            status: "passed",
-            executedAt: new Date().toISOString(),
-            evidence: {},
-          };
-
-          try {
-            // Execute the step action
-            await executeTestStep(playwrightClient, step);
-
-            // Wait briefly for page to settle
-            await new Promise((resolve) => setTimeout(resolve, 500));
-
-            // Capture evidence
-            if (runOptions.captureEvidence) {
-              const screenshot = await playwrightClient.screenshot();
-              const snapshot = await playwrightClient.snapshot();
-
-              const evidence = await workspaceManager.saveTestStepEvidence(
-                input.analysisId,
-                runId,
-                step.stepNumber,
-                {
-                  screenshot,
-                  snapshot: JSON.stringify(snapshot),
-                }
-              );
-
-              stepResult.evidence = evidence;
-            }
-
-            // Evaluate step if expected outcome specified
-            if (step.expected && samplingClient.hasSampling()) {
-              const snapshot = await playwrightClient.snapshot();
-
-              const evalResult = await samplingClient.createMessage({
-                systemPrompt:
-                  "You are evaluating whether a test step produced the expected outcome.",
-                userPrompt: buildTestEvaluationPrompt({
-                  testStep: `${step.action}${step.target ? ` on "${step.target}"` : ""}${step.value ? ` with value "${step.value}"` : ""}`,
-                  expectedOutcome: step.expected,
-                  actualState: snapshot.content.slice(0, 5000),
-                }),
-                schema: TestEvaluationSchema,
-                maxTokens: 1024,
-              });
-
-              if (evalResult.success && evalResult.data) {
-                if (!evalResult.data.passed) {
-                  stepResult.status = "failed";
-                  stepResult.actualResult = evalResult.data.reasoning;
-
-                  if (overallStatus === "passed") {
-                    overallStatus = "failed";
-                    failureStep = step.stepNumber;
-                  }
-
-                  stepLogger.warn("Step assertion failed", {
-                    stepNumber: step.stepNumber,
-                    reasoning: evalResult.data.reasoning,
-                  });
-
-                  if (runOptions.stopOnFailure) {
-                    stepResults.push(stepResult);
-                    break;
-                  }
-                }
-              }
-            }
-          } catch (error) {
-            const message =
-              error instanceof Error ? error.message : "Unknown error";
-
-            stepResult.status = "error";
-            stepResult.errorMessage = message;
-
-            if (overallStatus === "passed") {
-              overallStatus = "error";
-              failureStep = step.stepNumber;
-            }
-
-            stepLogger.error("Step execution error", {
-              stepNumber: step.stepNumber,
-              error: message,
-            });
-
-            // Retry if enabled
-            if (runOptions.retryFailedSteps) {
-              stepLogger.info("Retrying failed step", {
-                stepNumber: step.stepNumber,
-              });
-
-              try {
-                await executeTestStep(playwrightClient, step);
-                stepResult.status = "passed";
-                stepResult.errorMessage = undefined;
-
-                if (failureStep === step.stepNumber) {
-                  overallStatus = "passed";
-                  failureStep = undefined;
-                }
-              } catch (retryError) {
-                // Retry also failed
-                stepLogger.error("Step retry also failed", {
-                  stepNumber: step.stepNumber,
-                });
-
-                if (runOptions.stopOnFailure) {
-                  stepResults.push(stepResult);
-                  break;
-                }
-              }
-            } else if (runOptions.stopOnFailure) {
-              stepResults.push(stepResult);
-              break;
-            }
-          }
-
-          stepResults.push(stepResult);
-        }
-
-        // Mark skipped steps
-        const executedSteps = stepResults.map((r) => r.stepNumber);
-        for (const step of testCase.steps) {
-          if (!executedSteps.includes(step.stepNumber)) {
-            stepResults.push({
-              stepNumber: step.stepNumber,
-              status: "skipped",
-              executedAt: new Date().toISOString(),
-              evidence: {},
-            });
-          }
-        }
-
-        // Sort results by step number
-        stepResults.sort((a, b) => a.stepNumber - b.stepNumber);
-
-        // Update test run index
-        await workspaceManager.updateTestRunIndex(input.analysisId, runId, {
-          status: overallStatus,
-          completedAt: new Date().toISOString(),
-          steps: stepResults,
-        });
-
-        // Update workspace run reference
-        const updatedWorkspace = await workspaceManager.readWorkspaceIndex(
-          input.analysisId
-        );
-        const runRef = updatedWorkspace.runs.find((r) => r.runId === runId);
-        if (runRef) {
-          runRef.status = overallStatus;
-          runRef.completedAt = new Date().toISOString();
-          await workspaceManager.updateWorkspaceIndex(input.analysisId, {
-            runs: updatedWorkspace.runs,
-          });
-        }
-
-        await resourceManager.notifyListChanged();
-
-        // Generate result summary
-        const passedSteps = stepResults.filter((r) => r.status === "passed").length;
-        const failedSteps = stepResults.filter((r) => r.status === "failed").length;
-        const errorSteps = stepResults.filter((r) => r.status === "error").length;
-        const skippedSteps = stepResults.filter((r) => r.status === "skipped").length;
-
-        const result = {
-          runId,
-          analysisId: input.analysisId,
-          testCaseId: input.testCaseId,
-          testName: testCase.name,
-          status: overallStatus,
-          summary: {
-            totalSteps: testCase.steps.length,
-            passed: passedSteps,
-            failed: failedSteps,
-            error: errorSteps,
-            skipped: skippedSteps,
-          },
-          failureStep,
-          reportFilePath: join(runPath, "report.md"),
-          reportUri: `webtest://${input.analysisId}/runs/${runId}/report.md`,
-          stepResults: stepResults.map((r) => ({
-            stepNumber: r.stepNumber,
-            status: r.status,
-            evidence: r.evidence,
-            actualResult: r.actualResult,
-            errorMessage: r.errorMessage,
-          })),
-          nextSteps:
-            overallStatus === "passed"
-              ? ["All assertions passed!", "Run more test cases as needed"]
-              : [
-                  `Investigate failure at step ${failureStep}`,
-                  `Review evidence at ${stepResults.find((r) => r.stepNumber === failureStep)?.evidence.screenshotUri || "N/A"}`,
-                ],
-        };
-
-        stepLogger.info("Test execution completed", {
-          status: overallStatus,
-          passed: passedSteps,
-          failed: failedSteps,
-          error: errorSteps,
-        });
-
-        return {
-          content: [
-            {
-              type: "text",
-              text: JSON.stringify(result, null, 2),
-            },
-          ],
-        };
-      } catch (error) {
-        if (error instanceof CancellationError) {
-          runLogger.info("Test execution cancelled", {
-            requestId: error.requestId,
-          });
-
-          return {
-            content: [
-              {
-                type: "text",
-                text: JSON.stringify(
-                  {
-                    status: "cancelled",
-                    message: "Test execution was cancelled by user",
-                  },
-                  null,
-                  2
-                ),
-              },
-            ],
-          };
-        }
-
-        const message = error instanceof Error ? error.message : "Unknown error";
-        runLogger.error("Test execution failed", { error: message });
-
-        return {
-          content: [
-            {
-              type: "text",
-              text: `Error during test execution: ${message}`,
-            },
-          ],
-          isError: true,
-        };
-      } finally {
-        cancellationRegistry.unregister(requestId);
-      }
-    },
-  };
-}
-
-async function executeTestStep(
-  playwright: PlaywrightClient,
-  step: { action: string; target?: string; value?: string; element?: string; ref?: string }
-): Promise<void> {
-  const action = step.action.toLowerCase();
-
-  // Use element/ref if provided, otherwise fall back to target as the element description
-  // with an empty ref (some implementations may accept this)
-  const element = step.element || step.target || "";
-  const ref = step.ref || "";
-
-  // Parse the action to determine what to do
-  if (action.includes("navigate") || action.includes("go to")) {
-    const url = step.value || step.target || "";
-    if (url) {
-      await playwright.navigate(url);
-    }
-  } else if (action.includes("click")) {
-    if (element) {
-      await playwright.click(element, ref);
-    }
-  } else if (action.includes("type") || action.includes("enter")) {
-    if (element && step.value) {
-      await playwright.type(element, ref, step.value);
-    }
-  } else if (action.includes("fill")) {
-    if (element && step.value) {
-      await playwright.fill(element, ref, step.value);
-    }
-  } else if (action.includes("hover")) {
-    if (element) {
-      await playwright.hover(element, ref);
-    }
-  } else if (action.includes("select")) {
-    if (element && step.value) {
-      // Convert single value to array as required by the API
-      const values = step.value.includes(",")
-        ? step.value.split(",").map(v => v.trim())
-        : [step.value];
-      await playwright.select(element, ref, values);
-    }
-  } else if (action.includes("press")) {
-    if (step.value) {
-      // press only takes the key, no element targeting
-      await playwright.press(step.value);
-    }
-  } else if (action.includes("scroll")) {
-    const x = parseInt(step.target || "0", 10);
-    const y = parseInt(step.value || "0", 10);
-    await playwright.scroll(x, y);
-  } else if (action.includes("wait")) {
-    const ms = parseInt(step.value || "1000", 10);
-    await playwright.wait(ms);
-  } else {
-    // Default: try to interpret as click if element provided
-    if (element) {
-      await playwright.click(element, ref);
-    }
-  }
-}

package/src/tools/webtest/schemas.ts

@@ -1,201 +0,0 @@
-import { z } from "zod";
-
-// Common schemas used across webtest tools
-
-/**
- * Analysis ID schema - accepts either:
- * - Date-time format: YYYY-MM-DD_HH-mm (new format for workspace folders)
- * - UUID format (legacy support)
- */
-export const AnalysisIdSchema = z
-  .string()
-  .describe("Analysis workspace ID in format YYYY-MM-DD_HH-mm (e.g., 2025-12-18_17-05) or legacy UUID")
-  .refine(
-    (val) => {
-      // Match date-time format: YYYY-MM-DD_HH-mm
-      const dateTimePattern = /^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}$/;
-      // Match UUID format (for legacy support)
-      const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-      return dateTimePattern.test(val) || uuidPattern.test(val);
-    },
-    { message: "Invalid analysis ID. Expected format: YYYY-MM-DD_HH-mm or UUID" }
-  );
-
-/**
- * Crawl ID schema - accepts either:
- * - Date-time format with seconds: YYYY-MM-DD_HH-mm-ss (new format for crawl folders)
- * - UUID format (legacy support)
- */
-export const CrawlIdSchema = z
-  .string()
-  .describe("Crawl ID in format YYYY-MM-DD_HH-mm-ss (e.g., 2025-12-18_17-06-20) or legacy UUID")
-  .refine(
-    (val) => {
-      // Match date-time format with seconds: YYYY-MM-DD_HH-mm-ss
-      const dateTimePattern = /^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$/;
-      // Match UUID format (for legacy support)
-      const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-      return dateTimePattern.test(val) || uuidPattern.test(val);
-    },
-    { message: "Invalid crawl ID. Expected format: YYYY-MM-DD_HH-mm-ss or UUID" }
-  );
-
-export const AnalysisLimitsSchema = z.object({
-  maxSteps: z.number().int().min(1).max(1000).optional(),
-  maxMinutes: z.number().int().min(1).max(180).optional(),
-  maxPages: z.number().int().min(1).max(100).optional(),
-});
-
-export type AnalysisLimits = z.infer<typeof AnalysisLimitsSchema>;
-
-export const DomainAllowlistSchema = z.array(z.string()).optional();
-
-// Sampling response schemas
-
-/**
- * Args structure per tool (Microsoft Playwright MCP compatible):
- * - navigate: { url: string }
- * - click: { element: string, ref: string }
- * - type: { element: string, ref: string, text: string, submit?: boolean, slowly?: boolean }
- * - fill: { element: string, ref: string, value: string }
- * - hover: { element: string, ref: string }
- * - select: { element: string, ref: string, values: string[] }
- * - press: { key: string }  // No element targeting
- * - scroll: { x: number, y: number }
- * - wait: { ms: number }
- *
- * The `element` field is a human-readable description of the element.
- * The `ref` field is the exact element reference from the accessibility snapshot (e.g., "e1", "e3").
- */
-export const CrawlActionSchema = z.object({
-  actions: z.array(
-    z.object({
-      tool: z.enum([
-        "navigate",
-        "click",
-        "type",
-        "fill",
-        "hover",
-        "select",
-        "press",
-        "scroll",
-        "wait",
-      ]),
-      args: z.record(z.string(), z.any()),
-    })
-  ),
-  reasoning: z.string(),
-  goalProgress: z.string(),
-  goalComplete: z.boolean().optional(),
-  blocked: z.boolean().optional(),
-  blockedReason: z.string().optional(),
-  elicitationNeeded: z
-    .object({
-      type: z.enum([
-        "cookie_consent",
-        "modal_blocking",
-        "ambiguous_navigation",
-        "auth_required",
-      ]),
-      context: z.string(),
-      options: z
-        .array(
-          z.object({
-            url: z.string().optional(),
-            label: z.string(),
-          })
-        )
-        .optional(),
-    })
-    .optional(),
-});
-
-export type CrawlAction = z.infer<typeof CrawlActionSchema>;
-
-// Feature-based schemas
-
-export const FeatureSchema = z.object({
-  slug: z.string().describe("URL-safe identifier for the feature (kebab-case)"),
-  name: z.string().describe("Human-readable feature name"),
-  description: z.string().describe("Brief description of what this feature does"),
-  entities: z.array(z.string()).describe("Key entities/data types in this feature"),
-  entryPoints: z.array(z.string()).describe("URLs or navigation paths to access this feature"),
-});
-
-export type Feature = z.infer<typeof FeatureSchema>;
-
-export const FeaturesDiscoverySchema = z.object({
-  appPurpose: z.string().describe("Overall purpose of the application"),
-  appType: z.string().describe("Type of application (e.g., e-commerce, SaaS, content site)"),
-  features: z.array(FeatureSchema).describe("Distinct features/modules of the application"),
-  securityObservations: z.array(z.string()).optional(),
-  accessibilityObservations: z.array(z.string()).optional(),
-});
-
-export type FeaturesDiscovery = z.infer<typeof FeaturesDiscoverySchema>;
-
-export const FlowSchema = z.object({
-  id: z.string().describe("Unique flow identifier"),
-  name: z.string().describe("Human-readable flow name"),
-  description: z.string().describe("What this flow accomplishes"),
-  entryPoint: z.string().describe("Starting URL or navigation path"),
-  steps: z.array(z.string()).describe("Ordered list of steps in the flow"),
-});
-
-export type Flow = z.infer<typeof FlowSchema>;
-
-export const FlowsDiscoverySchema = z.object({
-  featureSlug: z.string().describe("The feature these flows belong to"),
-  flows: z.array(FlowSchema).describe("User flows within this feature"),
-  suggestedAssertions: z.array(z.string()).describe("Assertions that should hold for these flows"),
-});
-
-export type FlowsDiscovery = z.infer<typeof FlowsDiscoverySchema>;
-
-export const FeatureSlugSchema = z
-  .string()
-  .regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/, "Feature slug must be kebab-case (e.g., 'user-auth', 'shopping-cart')")
-  .describe("Feature identifier in kebab-case");
-
-export const TestCaseSchema = z.object({
-  id: z.string(),
-  name: z.string(),
-  purpose: z.string(),
-  category: z.enum(["happy_path", "edge_case", "error_handling", "boundary"]),
-  preconditions: z.array(z.string()),
-  steps: z.array(
-    z.object({
-      stepNumber: z.number().int(),
-      action: z.string(),
-      target: z.string().optional(), // Kept for backwards compatibility
-      value: z.string().optional(),
-      expected: z.string().optional(),
-      // Microsoft Playwright MCP compatible fields
-      element: z.string().optional(), // Human-readable element description
-      ref: z.string().optional(), // Element reference from accessibility snapshot (e.g., "e1", "e3")
-    })
-  ),
-  expectedOutcomes: z.array(z.string()),
-  tags: z.array(z.string()).optional(),
-});
-
-export type TestCase = z.infer<typeof TestCaseSchema>;
-
-export const TestGenerationSchema = z.object({
-  tests: z.array(TestCaseSchema),
-  coverage: z.object({
-    flowsCovered: z.array(z.string()),
-    estimatedCoverage: z.string(),
-  }),
-});
-
-export type TestGeneration = z.infer<typeof TestGenerationSchema>;
-
-export const TestEvaluationSchema = z.object({
-  passed: z.boolean(),
-  reasoning: z.string(),
-  evidence: z.array(z.string()),
-  discrepancies: z.array(z.string()).optional(),
-});
-
-export type TestEvaluation = z.infer<typeof TestEvaluationSchema>;