retestkit 1.4.1

package/src/tools/webtest/generate-tests.test.ts

@@ -0,0 +1,532 @@
+/**
+ * Unit Tests for webtest-generate_tests tool (Phase 6.7)
+ */
+
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { createGenerateTestsTool } from "./generate-tests.js";
+import {
+  createMockContext,
+  type MockContext,
+} from "../../test-utils/index.js";
+
+describe("webtest-generate_tests", () => {
+  let context: MockContext;
+  let tool: ReturnType<typeof createGenerateTestsTool>;
+
+  beforeEach(() => {
+    context = createMockContext();
+    tool = createGenerateTestsTool(() => context as any);
+
+    // Set up workspace with features and featureFlows
+    context.workspaceManager.readWorkspaceIndex = vi.fn().mockResolvedValue({
+      url: "https://shop.example.com",
+      domain: "shop.example.com",
+      focus: "Test checkout flow",
+      crawls: [],
+      features: {
+        createdAt: new Date().toISOString(),
+        featuresFilePath: "/test/features.md",
+        featuresUri: "webtest://test/features.md",
+        featureCount: 2,
+        features: [
+          {
+            slug: "product-browsing",
+            name: "Product Browsing",
+            description: "Browse and view products",
+            entities: ["Product"],
+            entryPoints: ["/products"],
+          },
+          {
+            slug: "checkout",
+            name: "Checkout",
+            description: "Complete purchase",
+            entities: ["Cart", "Order"],
+            entryPoints: ["/cart"],
+          },
+        ],
+      },
+      featureFlows: [
+        {
+          featureSlug: "product-browsing",
+          createdAt: new Date().toISOString(),
+          flowsFilePath: "/test/features/product-browsing/flows.md",
+          flowsUri: "webtest://test/features/product-browsing/flows.md",
+          flowCount: 1,
+        },
+        {
+          featureSlug: "checkout",
+          createdAt: new Date().toISOString(),
+          flowsFilePath: "/test/features/checkout/flows.md",
+          flowsUri: "webtest://test/features/checkout/flows.md",
+          flowCount: 1,
+        },
+      ],
+      limits: { maxSteps: 100, maxMinutes: 30, maxPages: 50 },
+    });
+
+    // Set up resource reading for features and flows
+    context.resourceManager.readResource = vi.fn().mockImplementation((uri: string) => {
+      if (uri.includes("features.md") && !uri.includes("features/")) {
+        return Promise.resolve({
+          text: "# Features\n\n## Product Browsing\n\nBrowse and view products\n\n## Checkout\n\nComplete purchase",
+        });
+      }
+      if (uri.includes("product-browsing/flows")) {
+        return Promise.resolve({
+          text: `---
+flows:
+  - id: browse-products
+    name: Browse Products
+    entryPoint: /products
+    description: User views products
+    steps:
+      - Navigate to products
+      - View list
+---
+
+# Product Browsing Flows
+
+## Browse Products
+...
+`,
+        });
+      }
+      if (uri.includes("checkout/flows")) {
+        return Promise.resolve({
+          text: `---
+flows:
+  - id: checkout
+    name: Checkout Flow
+    entryPoint: /cart
+    description: User completes purchase
+    steps:
+      - Add to cart
+      - Enter details
+      - Pay
+---
+
+# Checkout Flows
+
+## Checkout Flow
+...
+`,
+        });
+      }
+      return Promise.resolve({ text: "" });
+    });
+
+    // Set up sampling with mock test generation result
+    context.samplingClient.createMessage = vi.fn().mockResolvedValue({
+      success: true,
+      data: {
+        tests: [
+          {
+            id: "test-browse-001",
+            name: "Browse products successfully",
+            category: "happy_path",
+            purpose: "Verify user can browse product list",
+            preconditions: ["User is on home page"],
+            steps: [
+              {
+                stepNumber: 1,
+                action: "Click",
+                target: "a.products-link",
+                expected: "Products page loads",
+              },
+              {
+                stepNumber: 2,
+                action: "Verify",
+                expected: "Product list is visible",
+              },
+            ],
+            expectedOutcomes: ["Products page is displayed", "Products are listed"],
+            tags: ["browse", "products"],
+          },
+          {
+            id: "test-checkout-001",
+            name: "Complete checkout",
+            category: "happy_path",
+            purpose: "Verify user can complete checkout",
+            preconditions: ["User has item in cart"],
+            steps: [
+              {
+                stepNumber: 1,
+                action: "Navigate to",
+                target: "/cart",
+                expected: "Cart page loads",
+              },
+              {
+                stepNumber: 2,
+                action: "Click",
+                target: "button.checkout",
+                expected: "Checkout form appears",
+              },
+            ],
+            expectedOutcomes: ["Order is placed", "Confirmation shown"],
+            tags: ["checkout", "payment"],
+          },
+        ],
+        coverage: {
+          flows: ["browse-products", "checkout"],
+          assertionsCovered: 5,
+        },
+      },
+    });
+  });
+
+  describe("tool metadata", () => {
+    it("has correct name", () => {
+      expect(tool.name).toBe("webtest_generate_tests");
+    });
+
+    it("has a description", () => {
+      expect(tool.description).toBeDefined();
+      expect(tool.description.length).toBeGreaterThan(0);
+    });
+
+    it("has an input schema", () => {
+      expect(tool.inputSchema).toBeDefined();
+    });
+  });
+
+  describe("handler - validation", () => {
+    it("returns error for non-existent workspace", async () => {
+      context.workspaceManager.workspaceExists = vi.fn().mockResolvedValue(false);
+
+      const result = await tool.handler({
+        analysisId: "00000000-0000-0000-0000-000000000000",
+      });
+
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain("not found");
+    });
+
+    it("returns error when no features exist", async () => {
+      context.workspaceManager.readWorkspaceIndex = vi.fn().mockResolvedValue({
+        url: "https://example.com",
+        domain: "example.com",
+        crawls: [],
+        features: null,
+        limits: { maxSteps: 100, maxMinutes: 30, maxPages: 50 },
+      });
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain("No features");
+    });
+
+    it("returns error when features data fails to load", async () => {
+      context.resourceManager.readResource = vi.fn().mockRejectedValue(
+        new Error("File not found")
+      );
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain("Error loading");
+    });
+  });
+
+  describe("handler - test generation", () => {
+    it("calls sampling with test generation prompt", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(context.samplingClient.createMessage).toHaveBeenCalledWith(
+        expect.objectContaining({
+          systemPrompt: expect.stringContaining("QA engineer"),
+          userPrompt: expect.any(String),
+          schema: expect.any(Object),
+        })
+      );
+    });
+
+    it("saves generated tests to workspace", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(context.workspaceManager.saveTests).toHaveBeenCalledWith(
+        context.testAnalysisId,
+        expect.objectContaining({
+          markdown: expect.any(String),
+          frontmatter: expect.objectContaining({
+            tests: expect.any(Array),
+          }),
+        })
+      );
+    });
+
+    it("returns test count and URIs", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(result.isError).toBeFalsy();
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.testsUri).toBeDefined();
+      expect(content.testCount).toBeGreaterThan(0);
+    });
+
+    it("returns test summary with categories", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.categories).toBeDefined();
+      expect(content.testSummary).toBeInstanceOf(Array);
+    });
+
+    it("notifies about new resources", async () => {
+      vi.clearAllMocks();
+
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(context.resourceManager.notifyListChanged).toHaveBeenCalled();
+    });
+  });
+
+  describe("handler - strategies", () => {
+    it("accepts comprehensive strategy", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testStrategy: "comprehensive",
+      });
+
+      expect(result.isError).toBeFalsy();
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.strategy).toBe("comprehensive");
+    });
+
+    it("accepts happy_path strategy", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testStrategy: "happy_path",
+      });
+
+      expect(result.isError).toBeFalsy();
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.strategy).toBe("happy_path");
+    });
+
+    it("accepts edge_cases strategy", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testStrategy: "edge_cases",
+      });
+
+      expect(result.isError).toBeFalsy();
+    });
+
+    it("accepts critical_paths strategy", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+        testStrategy: "critical_paths",
+      });
+
+      expect(result.isError).toBeFalsy();
+    });
+  });
+
+  describe("handler - focus flows", () => {
+    it("filters flows when focusFlows provided", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        focusFlows: ["checkout"],
+      });
+
+      expect(context.samplingClient.createMessage).toHaveBeenCalled();
+      // The prompt should only include the checkout flow
+    });
+
+    it("uses all flows when focusFlows not provided", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(context.samplingClient.createMessage).toHaveBeenCalled();
+    });
+  });
+
+  describe("handler - max tests limit", () => {
+    it("respects maxTests parameter", async () => {
+      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
+        success: true,
+        data: {
+          tests: [
+            { id: "test-1", name: "Test 1", category: "happy_path", purpose: "P", preconditions: [], steps: [], expectedOutcomes: [] },
+            { id: "test-2", name: "Test 2", category: "happy_path", purpose: "P", preconditions: [], steps: [], expectedOutcomes: [] },
+            { id: "test-3", name: "Test 3", category: "happy_path", purpose: "P", preconditions: [], steps: [], expectedOutcomes: [] },
+          ],
+          coverage: { flows: [], assertionsCovered: 0 },
+        },
+      });
+
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+        maxTests: 2,
+      });
+
+      // Verify saved tests are limited
+      expect(context.workspaceManager.saveTests).toHaveBeenCalledWith(
+        context.testAnalysisId,
+        expect.objectContaining({
+          frontmatter: expect.objectContaining({
+            tests: expect.any(Array),
+          }),
+        })
+      );
+
+      const saveCall = context.workspaceManager.saveTests.mock.calls[0];
+      expect(saveCall[1].frontmatter.tests.length).toBeLessThanOrEqual(2);
+    });
+
+    it("uses default maxTests of 20 by calling sampling", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      // The tool calls sampling with a system prompt mentioning the limit
+      expect(context.samplingClient.createMessage).toHaveBeenCalled();
+    });
+  });
+
+  describe("handler - fallback mode", () => {
+    beforeEach(() => {
+      context.samplingClient.hasSampling = vi.fn().mockReturnValue(false);
+    });
+
+    it("returns prompt when sampling unavailable", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(result.isError).toBeFalsy();
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.needsManualInput).toBe(true);
+      expect(content.prompt).toBeDefined();
+    });
+
+    it("includes expected schema in fallback response", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.expectedResponseSchema).toBeDefined();
+    });
+  });
+
+  describe("handler - error handling", () => {
+    it("handles sampling failure", async () => {
+      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
+        success: false,
+        error: "LLM error",
+      });
+
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain("Error");
+    });
+
+    it("logs errors appropriately", async () => {
+      context.samplingClient.createMessage = vi.fn().mockRejectedValue(
+        new Error("Network error")
+      );
+
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(context.logger.error).toHaveBeenCalled();
+    });
+  });
+
+  describe("handler - markdown generation", () => {
+    it("generates markdown with test cases", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      const saveCall = context.workspaceManager.saveTests.mock.calls[0];
+      const markdown = saveCall[1].markdown;
+
+      expect(markdown).toContain("# Test Cases");
+      expect(markdown).toContain("## Overview");
+      expect(markdown).toContain("## Test Cases");
+    });
+
+    it("includes test steps in markdown", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      const saveCall = context.workspaceManager.saveTests.mock.calls[0];
+      const markdown = saveCall[1].markdown;
+
+      expect(markdown).toContain("**Steps**");
+    });
+
+    it("includes expected outcomes in markdown", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      const saveCall = context.workspaceManager.saveTests.mock.calls[0];
+      const markdown = saveCall[1].markdown;
+
+      expect(markdown).toContain("**Expected Outcomes**");
+    });
+  });
+
+  describe("handler - next steps", () => {
+    it("returns next steps with first test suggestion", async () => {
+      const result = await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      const content = JSON.parse(result.content[0].text!);
+      expect(content.nextSteps).toBeInstanceOf(Array);
+      expect(content.nextSteps.length).toBeGreaterThan(0);
+      expect(content.nextSteps.some((s: string) => s.includes("run_tests"))).toBe(true);
+    });
+  });
+
+  describe("handler - logging", () => {
+    it("logs test generation start", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(context.logger.info).toHaveBeenCalledWith(
+        expect.stringContaining("test generation"),
+        expect.any(Object)
+      );
+    });
+
+    it("logs completion with test count", async () => {
+      await tool.handler({
+        analysisId: context.testAnalysisId,
+      });
+
+      expect(context.logger.info).toHaveBeenCalledWith(
+        expect.stringContaining("completed"),
+        expect.objectContaining({
+          testCount: expect.any(Number),
+        })
+      );
+    });
+  });
+});