retestkit 1.4.1 → 1.5.0

package/src/tools/webtest/run-test-case.test.ts

@@ -1,659 +0,0 @@
-/**
- * Unit Tests for webtest_run_tests tool (Phase 7.8)
- */
-
-import { describe, it, expect, vi, beforeEach } from "vitest";
-import { createRunTestCaseTool } from "./run-test-case.js";
-import {
-  createMockContext,
-  type MockContext,
-} from "../../test-utils/index.js";
-
-describe("webtest_run_tests", () => {
-  let context: MockContext;
-  let tool: ReturnType<typeof createRunTestCaseTool>;
-
-  beforeEach(() => {
-    context = createMockContext();
-    tool = createRunTestCaseTool(() => context as any);
-
-    // Set up workspace with tests
-    context.workspaceManager.readWorkspaceIndex = vi.fn().mockResolvedValue({
-      url: "https://shop.example.com",
-      domain: "shop.example.com",
-      focus: "Test checkout flow",
-      crawls: [],
-      analysis: {
-        appAnalysisUri: "webtest://test/analysis/app-analysis.md",
-        flowsUri: "webtest://test/analysis/flows.md",
-      },
-      tests: {
-        testsUri: "webtest://test/tests/tests.md",
-        testCount: 2,
-      },
-      runs: [],
-      limits: { maxSteps: 100, maxMinutes: 30, maxPages: 50 },
-    });
-
-    // Set up resource reading for tests (markdown with YAML frontmatter)
-    context.resourceManager.readResource = vi.fn().mockResolvedValue({
-      text: `---
-tests:
-  - id: test-browse-001
-    name: Browse products successfully
-    category: happy_path
-    purpose: Verify user can browse product list
-    preconditions:
-      - User is on home page
-    steps:
-      - stepNumber: 1
-        action: Click
-        target: a.products-link
-        expected: Products page loads
-      - stepNumber: 2
-        action: Verify
-        expected: Product list is visible
-    expectedOutcomes:
-      - Products page is displayed
-    tags:
-      - browse
-  - id: test-checkout-001
-    name: Complete checkout
-    category: happy_path
-    purpose: Verify user can complete checkout
-    preconditions: []
-    steps:
-      - stepNumber: 1
-        action: Navigate to
-        target: /cart
-        expected: Cart page loads
-    expectedOutcomes:
-      - Order is placed
-    tags:
-      - checkout
----
-
-# Test Cases
-
-## Browse products successfully
-...
-`,
-    });
-
-    // Set up evaluation result
-    context.samplingClient.createMessage = vi.fn().mockResolvedValue({
-      success: true,
-      data: {
-        passed: true,
-        reasoning: "Expected outcome matched",
-        confidence: 0.95,
-      },
-    });
-
-    // Connect playwright
-    context.playwrightClient.isConnected = vi.fn().mockReturnValue(true);
-  });
-
-  describe("tool metadata", () => {
-    it("has correct name", () => {
-      expect(tool.name).toBe("webtest_run_test");
-    });
-
-    it("has a description", () => {
-      expect(tool.description).toBeDefined();
-      expect(tool.description.length).toBeGreaterThan(0);
-    });
-
-    it("has an input schema", () => {
-      expect(tool.inputSchema).toBeDefined();
-    });
-  });
-
-  describe("handler - validation", () => {
-    it("returns error for non-existent workspace", async () => {
-      context.workspaceManager.workspaceExists = vi.fn().mockResolvedValue(false);
-
-      const result = await tool.handler({
-        analysisId: "00000000-0000-0000-0000-000000000000",
-        testCaseId: "test-1",
-      });
-
-      expect(result.isError).toBe(true);
-      expect(result.content[0].text).toContain("not found");
-    });
-
-    it("returns error when no tests exist", async () => {
-      context.workspaceManager.readWorkspaceIndex = vi.fn().mockResolvedValue({
-        url: "https://example.com",
-        domain: "example.com",
-        crawls: [],
-        analysis: null,
-        tests: null,
-        runs: [],
-        limits: { maxSteps: 100, maxMinutes: 30, maxPages: 50 },
-      });
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-1",
-      });
-
-      expect(result.isError).toBe(true);
-      expect(result.content[0].text).toContain("No tests found");
-    });
-
-    it("returns error for non-existent test case", async () => {
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "non-existent-test",
-      });
-
-      expect(result.isError).toBe(true);
-      expect(result.content[0].text).toContain("not found");
-      expect(result.content[0].text).toContain("Available tests");
-    });
-
-    it("returns error when tests fail to load", async () => {
-      context.resourceManager.readResource = vi.fn().mockRejectedValue(
-        new Error("File not found")
-      );
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(result.isError).toBe(true);
-      expect(result.content[0].text).toContain("Error loading");
-    });
-  });
-
-  describe("handler - test execution", () => {
-    it("creates test run in workspace", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.workspaceManager.createTestRun).toHaveBeenCalledWith(
-        context.testAnalysisId,
-        expect.objectContaining({
-          testCaseId: "test-browse-001",
-          testName: "Browse products successfully",
-        })
-      );
-    });
-
-    it("connects to playwright if not connected", async () => {
-      context.playwrightClient.isConnected = vi.fn().mockReturnValue(false);
-
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.playwrightClient.connect).toHaveBeenCalled();
-    });
-
-    it("navigates to workspace URL first", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.playwrightClient.navigate).toHaveBeenCalledWith(
-        "https://shop.example.com"
-      );
-    });
-
-    it("executes click actions", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      // When no element/ref provided, falls back to target as element with empty ref
-      expect(context.playwrightClient.click).toHaveBeenCalledWith("a.products-link", "");
-    });
-
-    it("returns runId in response", async () => {
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(result.isError).toBeFalsy();
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.runId).toBeDefined();
-    });
-
-    it("returns test status", async () => {
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.status).toBeDefined();
-      expect(["passed", "failed", "error"]).toContain(content.status);
-    });
-
-    it("returns step results", async () => {
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.stepResults).toBeInstanceOf(Array);
-      expect(content.stepResults.length).toBeGreaterThan(0);
-    });
-
-    it("returns summary with step counts", async () => {
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.summary).toBeDefined();
-      expect(content.summary.totalSteps).toBeGreaterThan(0);
-      expect(content.summary.passed).toBeDefined();
-    });
-  });
-
-  describe("handler - evidence capture", () => {
-    it("captures screenshot at each step by default", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.playwrightClient.screenshot).toHaveBeenCalled();
-    });
-
-    it("captures snapshot at each step by default", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.playwrightClient.snapshot).toHaveBeenCalled();
-    });
-
-    it("saves evidence to workspace", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.workspaceManager.saveTestStepEvidence).toHaveBeenCalled();
-    });
-
-    it("skips evidence when captureEvidence is false", async () => {
-      vi.clearAllMocks();
-
-      // Need to re-mock after clear
-      context.playwrightClient.navigate = vi.fn().mockResolvedValue(undefined);
-
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-        runOptions: {
-          captureEvidence: false,
-          stopOnFailure: true,
-          retryFailedSteps: false,
-        },
-      });
-
-      expect(context.workspaceManager.saveTestStepEvidence).not.toHaveBeenCalled();
-    });
-  });
-
-  describe("handler - assertion evaluation", () => {
-    it("evaluates step assertions via sampling", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.samplingClient.createMessage).toHaveBeenCalled();
-    });
-
-    it("marks step as failed when assertion fails", async () => {
-      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
-        success: true,
-        data: {
-          passed: false,
-          reasoning: "Expected product list but saw error page",
-          confidence: 0.9,
-        },
-      });
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.status).toBe("failed");
-    });
-
-    it("continues without sampling when unavailable", async () => {
-      context.samplingClient.hasSampling = vi.fn().mockReturnValue(false);
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      // Should still pass (no assertion evaluation)
-      expect(result.isError).toBeFalsy();
-    });
-  });
-
-  describe("handler - stop on failure", () => {
-    it("stops on first failure by default", async () => {
-      context.playwrightClient.click = vi.fn()
-        .mockRejectedValueOnce(new Error("Element not found"))
-        .mockResolvedValue(undefined);
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.status).toBe("error");
-      expect(content.summary.skipped).toBeGreaterThan(0);
-    });
-
-    it("continues on failure when stopOnFailure is false", async () => {
-      context.playwrightClient.click = vi.fn()
-        .mockRejectedValueOnce(new Error("Element not found"))
-        .mockResolvedValue(undefined);
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-        runOptions: {
-          captureEvidence: true,
-          stopOnFailure: false,
-          retryFailedSteps: false,
-        },
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      // All steps should have been attempted
-      expect(content.summary.skipped).toBe(0);
-    });
-  });
-
-  describe("handler - retry failed steps", () => {
-    it("retries failed steps when enabled", async () => {
-      let callCount = 0;
-      context.playwrightClient.click = vi.fn().mockImplementation(() => {
-        callCount++;
-        if (callCount === 1) {
-          return Promise.reject(new Error("Transient error"));
-        }
-        return Promise.resolve();
-      });
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-        runOptions: {
-          captureEvidence: true,
-          stopOnFailure: true,
-          retryFailedSteps: true,
-        },
-      });
-
-      // Should have retried and passed
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.status).toBe("passed");
-    });
-  });
-
-  describe("handler - cancellation", () => {
-    it("handles cancellation gracefully", async () => {
-      const { CancellationError } = await import("../../progress/index.js");
-      context.cancellationRegistry.checkCancelled = vi.fn().mockImplementation((id) => {
-        throw new CancellationError(id);
-      });
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(result.isError).toBeFalsy();
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.status).toBe("cancelled");
-    });
-
-    it("registers for cancellation on start", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.cancellationRegistry.register).toHaveBeenCalled();
-    });
-
-    it("unregisters cancellation on completion", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.cancellationRegistry.unregister).toHaveBeenCalled();
-    });
-  });
-
-  describe("handler - progress reporting", () => {
-    it("emits progress for each step", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.progressEmitter.emit).toHaveBeenCalled();
-    });
-
-    it("includes step info in progress", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.progressEmitter.emit).toHaveBeenCalledWith(
-        expect.objectContaining({
-          progress: expect.any(Number),
-          total: expect.any(Number),
-          message: expect.stringContaining("Step"),
-        })
-      );
-    });
-  });
-
-  describe("handler - resource notifications", () => {
-    it("notifies on test run creation", async () => {
-      vi.clearAllMocks();
-
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.resourceManager.notifyListChanged).toHaveBeenCalled();
-    });
-  });
-
-  describe("handler - action execution", () => {
-    it("executes navigate actions", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-checkout-001",
-      });
-
-      expect(context.playwrightClient.navigate).toHaveBeenCalledWith("/cart");
-    });
-
-    it("handles type actions", async () => {
-      context.resourceManager.readResource = vi.fn().mockResolvedValue({
-        text: `---
-tests:
-  - id: test-type-001
-    name: Type in field
-    category: happy_path
-    purpose: Test typing
-    preconditions: []
-    steps:
-      - stepNumber: 1
-        action: Type
-        target: input.search
-        value: test query
-    expectedOutcomes:
-      - Text is entered
----
-
-# Test Cases
-...
-`,
-      });
-
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-type-001",
-      });
-
-      // When no element/ref provided, falls back to target as element with empty ref
-      expect(context.playwrightClient.type).toHaveBeenCalledWith("input.search", "", "test query");
-    });
-
-    it("handles fill actions", async () => {
-      context.resourceManager.readResource = vi.fn().mockResolvedValue({
-        text: `---
-tests:
-  - id: test-fill-001
-    name: Fill field
-    category: happy_path
-    purpose: Test filling
-    preconditions: []
-    steps:
-      - stepNumber: 1
-        action: Fill
-        target: input.email
-        value: test@example.com
-    expectedOutcomes:
-      - Field is filled
----
-
-# Test Cases
-...
-`,
-      });
-
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-fill-001",
-      });
-
-      // When no element/ref provided, falls back to target as element with empty ref
-      expect(context.playwrightClient.fill).toHaveBeenCalledWith("input.email", "", "test@example.com");
-    });
-  });
-
-  describe("handler - error handling", () => {
-    it("handles playwright connection errors", async () => {
-      context.playwrightClient.isConnected = vi.fn().mockReturnValue(false);
-      context.playwrightClient.connect = vi.fn().mockRejectedValue(
-        new Error("Connection failed")
-      );
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(result.isError).toBe(true);
-    });
-
-    it("handles step execution errors", async () => {
-      context.playwrightClient.click = vi.fn().mockRejectedValue(
-        new Error("Element not found")
-      );
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.status).toBe("error");
-      expect(content.stepResults[0].errorMessage).toBeDefined();
-    });
-  });
-
-  describe("handler - next steps", () => {
-    it("returns success next steps when test passes", async () => {
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.nextSteps).toBeInstanceOf(Array);
-      expect(content.nextSteps.some((s: string) => s.includes("passed"))).toBe(true);
-    });
-
-    it("returns investigation steps when test fails", async () => {
-      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
-        success: true,
-        data: {
-          passed: false,
-          reasoning: "Element not visible",
-          confidence: 0.9,
-        },
-      });
-
-      const result = await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      const content = JSON.parse(result.content[0].text!);
-      expect(content.nextSteps.some((s: string) => s.includes("Investigate"))).toBe(true);
-    });
-  });
-
-  describe("handler - logging", () => {
-    it("logs test execution start", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.logger.info).toHaveBeenCalledWith(
-        expect.stringContaining("test case"),
-        expect.any(Object)
-      );
-    });
-
-    it("logs completion with results", async () => {
-      await tool.handler({
-        analysisId: context.testAnalysisId,
-        testCaseId: "test-browse-001",
-      });
-
-      expect(context.logger.info).toHaveBeenCalledWith(
-        expect.stringContaining("completed"),
-        expect.objectContaining({
-          status: expect.any(String),
-        })
-      );
-    });
-  });
-});