retestkit 1.4.1

package/src/tools/webtest/run-test-case.test.ts

@@ -0,0 +1,659 @@
+/**
+ * Unit Tests for webtest_run_tests tool (Phase 7.8)
+ */
+
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { createRunTestCaseTool } from "./run-test-case.js";
+import {
+  createMockContext,
+  type MockContext,
+} from "../../test-utils/index.js";
+
+describe("webtest_run_tests", () => {
+  let context: MockContext;
+  let tool: ReturnType<typeof createRunTestCaseTool>;
+
+  beforeEach(() => {
+    context = createMockContext();
+    tool = createRunTestCaseTool(() => context as any);
+
+    // Set up workspace with tests
+    context.workspaceManager.readWorkspaceIndex = vi.fn().mockResolvedValue({
+      url: "https://shop.example.com",
+      domain: "shop.example.com",
+      focus: "Test checkout flow",
+      crawls: [],
+      analysis: {
+        appAnalysisUri: "webtest://test/analysis/app-analysis.md",
+        flowsUri: "webtest://test/analysis/flows.md",
+      },
+      tests: {
+        testsUri: "webtest://test/tests/tests.md",
+        testCount: 2,
+      },
+      runs: [],
+      limits: { maxSteps: 100, maxMinutes: 30, maxPages: 50 },
+    });
+
+    // Set up resource reading for tests (markdown with YAML frontmatter)
+    context.resourceManager.readResource = vi.fn().mockResolvedValue({
+      text: `---
+tests:
+  - id: test-browse-001
+    name: Browse products successfully
+    category: happy_path
+    purpose: Verify user can browse product list
+    preconditions:
+      - User is on home page
+    steps:
+      - stepNumber: 1
+        action: Click
+        target: a.products-link
+        expected: Products page loads
+      - stepNumber: 2
+        action: Verify
+        expected: Product list is visible
+    expectedOutcomes:
+      - Products page is displayed
+    tags:
+      - browse
+  - id: test-checkout-001
+    name: Complete checkout
+    category: happy_path
+    purpose: Verify user can complete checkout
+    preconditions: []
+    steps:
+      - stepNumber: 1
+        action: Navigate to
+        target: /cart
+        expected: Cart page loads
+    expectedOutcomes:
+      - Order is placed
+    tags:
+      - checkout
+---
+
+# Test Cases
+
+## Browse products successfully
+...
+`,
+    });
+
+    // Set up evaluation result
+    context.samplingClient.createMessage = vi.fn().mockResolvedValue({
+      success: true,
+      data: {
+        passed: true,
+        reasoning: "Expected outcome matched",
+        confidence: 0.95,
+      },
+    });
+
+    // Connect playwright
+    context.playwrightClient.isConnected = vi.fn().mockReturnValue(true);
+  });
+
+  describe("tool metadata", () => {
+    it("has correct name", () => {
+      expect(tool.name).toBe("webtest_run_test");
+    });
+
+    it("has a description", () => {
+      expect(tool.description).toBeDefined();
+      expect(tool.description.length).toBeGreaterThan(0);
+    });
+
+    it("has an input schema", () => {
+      expect(tool.inputSchema).toBeDefined();
+    });
+  });
+
+  describe("handler - validation", () => {
+    it("returns error for non-existent workspace", async () => {
+      context.workspaceManager.workspaceExists = vi.fn().mockResolvedValue(false);
+
+      const result = await tool.handler({
+        analysisId: "00000000-0000-0000-0000-000000000000",
+        testCaseId: "test-1",
+      });
+
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain("not found");
+    });
+
+    it("returns error when no tests exist", async () => {
+      context.workspaceManager.readWorkspaceIndex = vi.fn().mockResolvedValue({
+        url: "https://example.com",
+        domain: "example.com",
+        crawls: [],
+        analysis: null,
+        tests: null,
+        runs: [],
+        limits: { maxSteps: 100, maxMinutes: 30, maxPages: 50 },
+      });
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-1",
+      });
+
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain("No tests found");
+    });
+
+    it("returns error for non-existent test case", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "non-existent-test",
+      });
+
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain("not found");
+      expect(result.content[0].text).toContain("Available tests");
+    });
+
+    it("returns error when tests fail to load", async () => {
+      context.resourceManager.readResource = vi.fn().mockRejectedValue(
+        new Error("File not found")
+      );
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain("Error loading");
+    });
+  });
+
+  describe("handler - test execution", () => {
+    it("creates test run in workspace", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.workspaceManager.createTestRun).toHaveBeenCalledWith(
+        context.testAnalysisId,
+        expect.objectContaining({
+          testCaseId: "test-browse-001",
+          testName: "Browse products successfully",
+        })
+      );
+    });
+
+    it("connects to playwright if not connected", async () => {
+      context.playwrightClient.isConnected = vi.fn().mockReturnValue(false);
+
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.playwrightClient.connect).toHaveBeenCalled();
+    });
+
+    it("navigates to workspace URL first", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.playwrightClient.navigate).toHaveBeenCalledWith(
+        "https://shop.example.com"
+      );
+    });
+
+    it("executes click actions", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      // When no element/ref provided, falls back to target as element with empty ref
+      expect(context.playwrightClient.click).toHaveBeenCalledWith("a.products-link", "");
+    });
+
+    it("returns runId in response", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(result.isError).toBeFalsy();
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.runId).toBeDefined();
+    });
+
+    it("returns test status", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.status).toBeDefined();
+      expect(["passed", "failed", "error"]).toContain(content.status);
+    });
+
+    it("returns step results", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.stepResults).toBeInstanceOf(Array);
+      expect(content.stepResults.length).toBeGreaterThan(0);
+    });
+
+    it("returns summary with step counts", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.summary).toBeDefined();
+      expect(content.summary.totalSteps).toBeGreaterThan(0);
+      expect(content.summary.passed).toBeDefined();
+    });
+  });
+
+  describe("handler - evidence capture", () => {
+    it("captures screenshot at each step by default", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.playwrightClient.screenshot).toHaveBeenCalled();
+    });
+
+    it("captures snapshot at each step by default", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.playwrightClient.snapshot).toHaveBeenCalled();
+    });
+
+    it("saves evidence to workspace", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.workspaceManager.saveTestStepEvidence).toHaveBeenCalled();
+    });
+
+    it("skips evidence when captureEvidence is false", async () => {
+      vi.clearAllMocks();
+
+      // Need to re-mock after clear
+      context.playwrightClient.navigate = vi.fn().mockResolvedValue(undefined);
+
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+        runOptions: {
+          captureEvidence: false,
+          stopOnFailure: true,
+          retryFailedSteps: false,
+        },
+      });
+
+      expect(context.workspaceManager.saveTestStepEvidence).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("handler - assertion evaluation", () => {
+    it("evaluates step assertions via sampling", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.samplingClient.createMessage).toHaveBeenCalled();
+    });
+
+    it("marks step as failed when assertion fails", async () => {
+      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
+        success: true,
+        data: {
+          passed: false,
+          reasoning: "Expected product list but saw error page",
+          confidence: 0.9,
+        },
+      });
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.status).toBe("failed");
+    });
+
+    it("continues without sampling when unavailable", async () => {
+      context.samplingClient.hasSampling = vi.fn().mockReturnValue(false);
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      // Should still pass (no assertion evaluation)
+      expect(result.isError).toBeFalsy();
+    });
+  });
+
+  describe("handler - stop on failure", () => {
+    it("stops on first failure by default", async () => {
+      context.playwrightClient.click = vi.fn()
+        .mockRejectedValueOnce(new Error("Element not found"))
+        .mockResolvedValue(undefined);
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.status).toBe("error");
+      expect(content.summary.skipped).toBeGreaterThan(0);
+    });
+
+    it("continues on failure when stopOnFailure is false", async () => {
+      context.playwrightClient.click = vi.fn()
+        .mockRejectedValueOnce(new Error("Element not found"))
+        .mockResolvedValue(undefined);
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+        runOptions: {
+          captureEvidence: true,
+          stopOnFailure: false,
+          retryFailedSteps: false,
+        },
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      // All steps should have been attempted
+      expect(content.summary.skipped).toBe(0);
+    });
+  });
+
+  describe("handler - retry failed steps", () => {
+    it("retries failed steps when enabled", async () => {
+      let callCount = 0;
+      context.playwrightClient.click = vi.fn().mockImplementation(() => {
+        callCount++;
+        if (callCount === 1) {
+          return Promise.reject(new Error("Transient error"));
+        }
+        return Promise.resolve();
+      });
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+        runOptions: {
+          captureEvidence: true,
+          stopOnFailure: true,
+          retryFailedSteps: true,
+        },
+      });
+
+      // Should have retried and passed
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.status).toBe("passed");
+    });
+  });
+
+  describe("handler - cancellation", () => {
+    it("handles cancellation gracefully", async () => {
+      const { CancellationError } = await import("../../progress/index.js");
+      context.cancellationRegistry.checkCancelled = vi.fn().mockImplementation((id) => {
+        throw new CancellationError(id);
+      });
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(result.isError).toBeFalsy();
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.status).toBe("cancelled");
+    });
+
+    it("registers for cancellation on start", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.cancellationRegistry.register).toHaveBeenCalled();
+    });
+
+    it("unregisters cancellation on completion", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.cancellationRegistry.unregister).toHaveBeenCalled();
+    });
+  });
+
+  describe("handler - progress reporting", () => {
+    it("emits progress for each step", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.progressEmitter.emit).toHaveBeenCalled();
+    });
+
+    it("includes step info in progress", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.progressEmitter.emit).toHaveBeenCalledWith(
+        expect.objectContaining({
+          progress: expect.any(Number),
+          total: expect.any(Number),
+          message: expect.stringContaining("Step"),
+        })
+      );
+    });
+  });
+
+  describe("handler - resource notifications", () => {
+    it("notifies on test run creation", async () => {
+      vi.clearAllMocks();
+
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.resourceManager.notifyListChanged).toHaveBeenCalled();
+    });
+  });
+
+  describe("handler - action execution", () => {
+    it("executes navigate actions", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-checkout-001",
+      });
+
+      expect(context.playwrightClient.navigate).toHaveBeenCalledWith("/cart");
+    });
+
+    it("handles type actions", async () => {
+      context.resourceManager.readResource = vi.fn().mockResolvedValue({
+        text: `---
+tests:
+  - id: test-type-001
+    name: Type in field
+    category: happy_path
+    purpose: Test typing
+    preconditions: []
+    steps:
+      - stepNumber: 1
+        action: Type
+        target: input.search
+        value: test query
+    expectedOutcomes:
+      - Text is entered
+---
+
+# Test Cases
+...
+`,
+      });
+
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-type-001",
+      });
+
+      // When no element/ref provided, falls back to target as element with empty ref
+      expect(context.playwrightClient.type).toHaveBeenCalledWith("input.search", "", "test query");
+    });
+
+    it("handles fill actions", async () => {
+      context.resourceManager.readResource = vi.fn().mockResolvedValue({
+        text: `---
+tests:
+  - id: test-fill-001
+    name: Fill field
+    category: happy_path
+    purpose: Test filling
+    preconditions: []
+    steps:
+      - stepNumber: 1
+        action: Fill
+        target: input.email
+        value: test@example.com
+    expectedOutcomes:
+      - Field is filled
+---
+
+# Test Cases
+...
+`,
+      });
+
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-fill-001",
+      });
+
+      // When no element/ref provided, falls back to target as element with empty ref
+      expect(context.playwrightClient.fill).toHaveBeenCalledWith("input.email", "", "test@example.com");
+    });
+  });
+
+  describe("handler - error handling", () => {
+    it("handles playwright connection errors", async () => {
+      context.playwrightClient.isConnected = vi.fn().mockReturnValue(false);
+      context.playwrightClient.connect = vi.fn().mockRejectedValue(
+        new Error("Connection failed")
+      );
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(result.isError).toBe(true);
+    });
+
+    it("handles step execution errors", async () => {
+      context.playwrightClient.click = vi.fn().mockRejectedValue(
+        new Error("Element not found")
+      );
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.status).toBe("error");
+      expect(content.stepResults[0].errorMessage).toBeDefined();
+    });
+  });
+
+  describe("handler - next steps", () => {
+    it("returns success next steps when test passes", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.nextSteps).toBeInstanceOf(Array);
+      expect(content.nextSteps.some((s: string) => s.includes("passed"))).toBe(true);
+    });
+
+    it("returns investigation steps when test fails", async () => {
+      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
+        success: true,
+        data: {
+          passed: false,
+          reasoning: "Element not visible",
+          confidence: 0.9,
+        },
+      });
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.nextSteps.some((s: string) => s.includes("Investigate"))).toBe(true);
+    });
+  });
+
+  describe("handler - logging", () => {
+    it("logs test execution start", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.logger.info).toHaveBeenCalledWith(
+        expect.stringContaining("test case"),
+        expect.any(Object)
+      );
+    });
+
+    it("logs completion with results", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        testCaseId: "test-browse-001",
+      });
+
+      expect(context.logger.info).toHaveBeenCalledWith(
+        expect.stringContaining("completed"),
+        expect.objectContaining({
+          status: expect.any(String),
+        })
+      );
+    });
+  });
+});