retestkit 1.4.1

package/src/tools/webtest/crawl.ts

@@ -0,0 +1,901 @@
+import { z } from "zod";
+import { join } from "node:path";
+import type { McpTool, ToolResult } from "../../types/tool.js";
+import type { ServerContext } from "../../types/context.js";
+import type { WorkspaceManager, CrawlCheckpoint, CrawlIndex } from "../../workspace/index.js";
+import type { PlaywrightClient } from "../../playwright-client/index.js";
+import type { SamplingClient } from "../../sampling/index.js";
+import type { ElicitationClient, ElicitationType } from "../../elicitation/index.js";
+import type { CancellationRegistry, ProgressEmitter } from "../../progress/index.js";
+import type { SecurityValidator } from "../../security/index.js";
+import type { ResourceManager } from "../../resources/index.js";
+import { CancellationError, calculateBudgetStatus } from "../../progress/index.js";
+import { buildCrawlActionPrompt } from "../../sampling/prompts.js";
+import { CrawlActionSchema, AnalysisIdSchema, type CrawlAction } from "./schemas.js";
+import { createDomSignature } from "../../security/index.js";
+
+export const crawlInputSchema = z.object({
+  analysisId: AnalysisIdSchema,
+  goal: z.string().min(1, "Goal is required").describe(
+    "What you want to discover or test (e.g., 'Find all navigation paths', 'Explore the checkout flow')"
+  ),
+  strategy: z
+    .enum(["breadth_first", "depth_first", "goal_directed"])
+    .default("goal_directed")
+    .describe("Crawling strategy"),
+  limits: z
+    .object({
+      maxSteps: z.number().int().min(1).max(1000).optional(),
+      maxMinutes: z.number().int().min(1).max(180).optional(),
+      maxPages: z.number().int().min(1).max(100).optional(),
+    })
+    .optional()
+    .describe("Override workspace limits for this crawl"),
+  artifacts: z
+    .object({
+      captureScreenshots: z.boolean().default(true),
+      captureSnapshots: z.boolean().default(true),
+      captureDom: z.boolean().default(true),
+    })
+    .optional()
+    .describe("What artifacts to capture"),
+  resume: z
+    .boolean()
+    .default(false)
+    .describe("Resume from the last checkpoint if available"),
+  manualNextActions: z
+    .array(
+      z.object({
+        tool: z.string(),
+        args: z.record(z.string(), z.any()),
+      })
+    )
+    .optional()
+    .describe("Manual actions to execute (for fallback mode when sampling unavailable)"),
+});
+
+export type CrawlInput = z.infer<typeof crawlInputSchema>;
+
+interface CrawlState {
+  step: number;
+  currentUrl: string;
+  startUrl: string;
+  visitedUrls: Set<string>;
+  actionHistory: string[];
+  goalProgress: string;
+  loopDetection: {
+    domSignatures: Map<string, number>;
+    urlVisits: Map<string, number>;
+    recentActions: string[];
+  };
+  /** Track if navigation to start URL was blocked (for prompt feedback) */
+  navigationBlocked?: {
+    url: string;
+    reason: string;
+  };
+}
+
+export function createCrawlTool(
+  getContext: () => ServerContext & {
+    workspaceManager: WorkspaceManager;
+    playwrightClient: PlaywrightClient;
+    samplingClient: SamplingClient;
+    elicitationClient: ElicitationClient;
+    cancellationRegistry: CancellationRegistry;
+    progressEmitter: ProgressEmitter;
+    securityValidator: SecurityValidator;
+    resourceManager: ResourceManager;
+  }
+): McpTool<CrawlInput> {
+  return {
+    name: "webtest_crawl_app",
+    description: `Perform goal-directed exploration of a web application.
+
+This tool crawls the target application using AI-powered decision making:
+- Navigates pages based on the specified goal
+- Captures screenshots, snapshots, and DOM at each step
+- Detects and handles obstacles (cookie banners, modals)
+- Supports checkpointing for resume after interruption
+- Prevents infinite loops through multiple detection mechanisms
+
+Progress is reported throughout, and the operation can be cancelled.
+If sampling is unavailable, returns a prompt for manual execution.`,
+
+    inputSchema: crawlInputSchema,
+
+    async handler(input: CrawlInput): Promise<ToolResult> {
+      const ctx = getContext();
+      const {
+        logger,
+        config,
+        workspaceManager,
+        playwrightClient,
+        samplingClient,
+        elicitationClient,
+        cancellationRegistry,
+        progressEmitter,
+        securityValidator,
+        resourceManager,
+      } = ctx;
+
+      const requestId = `crawl-${input.analysisId}-${Date.now()}`;
+      const crawlLogger = logger.withCorrelation({
+        analysisId: input.analysisId,
+        requestId,
+      });
+
+      crawlLogger.info("Starting crawl", { goal: input.goal, strategy: input.strategy });
+
+      // Register for cancellation
+      cancellationRegistry.register(requestId);
+
+      try {
+        // Validate workspace exists
+        if (!(await workspaceManager.workspaceExists(input.analysisId))) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Error: Analysis workspace "${input.analysisId}" not found. Run webtest_init first.`,
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        const workspace = await workspaceManager.readWorkspaceIndex(input.analysisId);
+        const allowedDomains = [workspace.domain];
+
+        // Determine limits
+        const limits = {
+          maxSteps: input.limits?.maxSteps ?? workspace.limits.maxSteps,
+          maxMinutes: input.limits?.maxMinutes ?? workspace.limits.maxMinutes,
+          maxPages: input.limits?.maxPages ?? workspace.limits.maxPages,
+        };
+
+        // Create crawl entry
+        const { crawlId, crawlPath } = await workspaceManager.createCrawl(
+          input.analysisId,
+          { goal: input.goal, strategy: input.strategy, limits }
+        );
+
+        const iterationLogger = crawlLogger.withCorrelation({ crawlId });
+        iterationLogger.info("Crawl created", { crawlId, limits });
+
+        // Notify about new resource
+        await resourceManager.notifyListChanged();
+
+        // Initialize state
+        let state: CrawlState;
+        const startTime = Date.now();
+
+        // Check for resume
+        if (input.resume) {
+          const checkpoint = await workspaceManager.loadCheckpoint(
+            input.analysisId,
+            crawlId
+          );
+
+          if (checkpoint && checkpoint.canResume) {
+            iterationLogger.info("Resuming from checkpoint", {
+              step: checkpoint.step,
+              hasLoopDetection: !!checkpoint.loopDetection,
+            });
+
+            // Restore loop detection state if available, otherwise start fresh
+            const loopDetection = checkpoint.loopDetection
+              ? {
+                  domSignatures: new Map(checkpoint.loopDetection.domSignatures),
+                  urlVisits: new Map(checkpoint.loopDetection.urlVisits),
+                  recentActions: [...checkpoint.loopDetection.recentActions],
+                }
+              : {
+                  domSignatures: new Map<string, number>(),
+                  urlVisits: new Map<string, number>(),
+                  recentActions: [] as string[],
+                };
+
+            if (!checkpoint.loopDetection) {
+              iterationLogger.warn("Resuming without loop detection history - context may be limited");
+            }
+
+            state = {
+              step: checkpoint.step,
+              currentUrl: checkpoint.currentUrl,
+              startUrl: checkpoint.startUrl || workspace.url,
+              visitedUrls: new Set(checkpoint.visitedUrls),
+              actionHistory: [],
+              goalProgress: checkpoint.goalProgress,
+              loopDetection,
+            };
+          } else {
+            state = createInitialState(workspace.url);
+          }
+        } else {
+          state = createInitialState(workspace.url);
+        }
+
+        // Ensure Playwright is connected
+        if (!playwrightClient.isConnected()) {
+          await playwrightClient.connect();
+        }
+
+        // Check current browser URL before navigating
+        // This prevents unnecessary navigation that would disrupt an existing browser session
+        let shouldNavigate = true;
+        try {
+          const currentSnapshot = await playwrightClient.snapshot();
+          const currentBrowserUrl = currentSnapshot.url;
+
+          // Skip navigation if browser is already on ANY page within the allowed domain
+          // This preserves the current browser state (e.g., logged in, items in cart)
+          if (currentBrowserUrl && currentBrowserUrl !== "about:blank") {
+            const currentDomain = new URL(currentBrowserUrl).hostname;
+            const targetDomain = new URL(state.startUrl).hostname;
+
+            if (currentDomain === targetDomain) {
+              // Browser is already on the target domain - don't navigate at all
+              // This preserves login state, cart contents, etc.
+              iterationLogger.info("Browser already on target domain, skipping navigation", {
+                currentBrowserUrl,
+                startUrl: state.startUrl,
+              });
+              shouldNavigate = false;
+              state.currentUrl = currentBrowserUrl;
+            }
+          }
+        } catch (e) {
+          // If we can't get current URL, proceed with navigation
+          iterationLogger.debug("Could not check current browser URL, will navigate", {
+            error: e instanceof Error ? e.message : "Unknown error",
+          });
+        }
+
+        if (shouldNavigate) {
+          // Navigate to starting URL
+          await playwrightClient.navigate(state.currentUrl);
+          iterationLogger.info("Navigated to starting URL", { url: state.currentUrl });
+        }
+
+        // Main crawl loop
+        let goalComplete = false;
+        let blocked = false;
+        let blockedReason: string | undefined;
+
+        while (!goalComplete && !blocked) {
+          // Check cancellation
+          cancellationRegistry.checkCancelled(requestId);
+
+          // Check budget
+          const budget = calculateBudgetStatus({
+            stepsUsed: state.step,
+            maxSteps: limits.maxSteps,
+            startTime,
+            maxMinutes: limits.maxMinutes,
+            pagesVisited: state.visitedUrls.size,
+            maxPages: limits.maxPages,
+          });
+
+          if (budget.isExhausted) {
+            iterationLogger.info("Budget exhausted", { reason: budget.exhaustedReason });
+            break;
+          }
+
+          // Emit progress
+          progressEmitter.emit({
+            progressToken: requestId,
+            progress: state.step,
+            total: limits.maxSteps,
+            message: `Step ${state.step}: ${state.goalProgress || "Exploring..."}`,
+            budgetStatus: budget,
+          });
+
+          // Capture current state
+          const snapshot = await playwrightClient.snapshot();
+          const screenshot = await playwrightClient.screenshot();
+
+          state.currentUrl = snapshot.url;
+
+          // Track URL visits for loop detection
+          const urlVisits = state.loopDetection.urlVisits.get(state.currentUrl) || 0;
+          state.loopDetection.urlVisits.set(state.currentUrl, urlVisits + 1);
+
+          // Check for URL cycle
+          if (urlVisits >= 3) {
+            iterationLogger.warn("URL cycle detected", {
+              url: state.currentUrl,
+              visits: urlVisits + 1,
+            });
+          }
+
+          // Check DOM signature for same-state loop (include URL path for semantic differentiation)
+          let urlPath: string | undefined;
+          try {
+            urlPath = new URL(state.currentUrl).pathname;
+          } catch {
+            // If URL parsing fails, try to extract path from the string or skip it
+            const pathMatch = state.currentUrl.match(/^(?:https?:\/\/[^\/]+)?(\/.*)$/);
+            urlPath = pathMatch?.[1];
+          }
+          const domSignature = createDomSignature(snapshot.content, urlPath);
+          const sigRepeats = state.loopDetection.domSignatures.get(domSignature) || 0;
+          state.loopDetection.domSignatures.set(domSignature, sigRepeats + 1);
+
+          if (sigRepeats >= 3) {
+            iterationLogger.warn("Same-state loop detected", {
+              signature: domSignature,
+              repeats: sigRepeats + 1,
+            });
+          }
+
+          // Save page artifacts
+          const artifacts = input.artifacts ?? {
+            captureScreenshots: true,
+            captureSnapshots: true,
+            captureDom: true,
+          };
+
+          if (
+            artifacts.captureScreenshots ||
+            artifacts.captureSnapshots ||
+            artifacts.captureDom
+          ) {
+            await workspaceManager.savePage(input.analysisId, crawlId, {
+              url: state.currentUrl,
+              title: snapshot.title,
+              snapshot: JSON.stringify(snapshot),
+              screenshot,
+              dom: snapshot.content,
+            });
+
+            state.visitedUrls.add(state.currentUrl);
+            await resourceManager.notifyListChanged();
+          }
+
+          // Check for injection attempts in page content
+          const injectionCheck = securityValidator.detectInjectionAttempt(
+            snapshot.content
+          );
+          if (injectionCheck.detected) {
+            iterationLogger.warn("Injection attempt detected in page content", {
+              type: injectionCheck.type,
+              evidence: injectionCheck.evidence,
+            });
+            // Don't act on it, but include warning in sampling prompt
+          }
+
+          // Handle manual mode (no sampling)
+          if (input.manualNextActions && input.manualNextActions.length > 0) {
+            const action = input.manualNextActions.shift()!;
+
+            // Validate action
+            const actionValidation = securityValidator.validateAction(
+              action,
+              allowedDomains
+            );
+            if (!actionValidation.valid) {
+              return {
+                content: [
+                  {
+                    type: "text",
+                    text: `Security error: ${actionValidation.reason}`,
+                  },
+                ],
+                isError: true,
+              };
+            }
+
+            await executeAction(playwrightClient, action, iterationLogger);
+            state.step++;
+            continue;
+          }
+
+          // Calculate budget percentage for flow progress
+          const budgetUsedPercent = Math.round((state.step / limits.maxSteps) * 100);
+
+          // Build common prompt parameters
+          const promptParams = {
+            goal: input.goal,
+            currentUrl: state.currentUrl,
+            pageSnapshot: snapshot.content.slice(0, 10000),
+            actionHistory: state.actionHistory,
+            allowedDomains,
+            startUrl: state.startUrl,
+            flowProgress: {
+              currentStep: state.step,
+              totalSteps: limits.maxSteps,
+              budgetUsedPercent,
+              previousGoalProgress: state.goalProgress,
+            },
+            loopState: {
+              domSignatureRepeats: sigRepeats,
+              urlVisitCount: Object.fromEntries(state.loopDetection.urlVisits),
+              lastActions: state.loopDetection.recentActions,
+            },
+            navigationBlocked: state.navigationBlocked,
+          };
+
+          // Clear navigation blocked after including in prompt
+          state.navigationBlocked = undefined;
+
+          // Request next action via sampling
+          if (!samplingClient.hasSampling()) {
+            // Return prompt for manual execution
+            const prompt = buildCrawlActionPrompt(promptParams);
+
+            const crawlIndex = await workspaceManager.readCrawlIndex(
+              input.analysisId,
+              crawlId
+            );
+
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: JSON.stringify(
+                    {
+                      needsManualInput: true,
+                      crawlId,
+                      step: state.step,
+                      currentUrl: state.currentUrl,
+                      prompt,
+                      expectedResponseSchema: CrawlActionSchema._def,
+                      instructions:
+                        "Execute this prompt with your LLM and call webtest_crawl_app again with manualNextActions",
+                      partialResults: {
+                        pagesVisited: crawlIndex.pages.length,
+                        stepsCompleted: state.step,
+                      },
+                    },
+                    null,
+                    2
+                  ),
+                },
+              ],
+            };
+          }
+
+          // Get action from sampling
+          const samplingResult = await samplingClient.createMessage({
+            systemPrompt: `You are analyzing a web application to achieve the following goal: ${input.goal}`,
+            userPrompt: buildCrawlActionPrompt(promptParams),
+            schema: CrawlActionSchema,
+            maxTokens: 2048,
+          });
+
+          if (!samplingResult.success || !samplingResult.data) {
+            iterationLogger.error("Sampling failed", {
+              error: samplingResult.error,
+            });
+            blocked = true;
+            blockedReason = `Sampling error: ${samplingResult.error}`;
+            break;
+          }
+
+          const actionResponse = samplingResult.data;
+          state.goalProgress = actionResponse.goalProgress;
+
+          // Check if goal is complete
+          if (actionResponse.goalComplete) {
+            goalComplete = true;
+            iterationLogger.info("Goal completed", {
+              reasoning: actionResponse.reasoning,
+            });
+            break;
+          }
+
+          // Check if blocked
+          if (actionResponse.blocked) {
+            blocked = true;
+            blockedReason = actionResponse.blockedReason;
+            iterationLogger.info("Crawl blocked", { reason: blockedReason });
+            break;
+          }
+
+          // Handle elicitation if needed
+          if (actionResponse.elicitationNeeded) {
+            const elicitResult = await handleElicitation(
+              elicitationClient,
+              actionResponse.elicitationNeeded,
+              iterationLogger
+            );
+
+            if (elicitResult.cancelled) {
+              blocked = true;
+              blockedReason = "User cancelled at elicitation prompt";
+              break;
+            }
+
+            // Use elicitation result to modify action if needed
+            if (
+              elicitResult.selectedValue === "stop" ||
+              elicitResult.selectedValue === "reject"
+            ) {
+              blocked = true;
+              blockedReason = `User chose: ${elicitResult.selectedValue}`;
+              break;
+            }
+          }
+
+          // Execute actions
+          for (const action of actionResponse.actions) {
+            // Check cancellation before each action
+            cancellationRegistry.checkCancelled(requestId);
+
+            // Navigation guard: block navigation to start URL after initial steps
+            if (action.tool === "navigate" && state.step >= 3) {
+              const targetUrl = action.args.url as string;
+              // Check if navigating back to start URL
+              if (isStartUrlNavigation(targetUrl, state.startUrl)) {
+                iterationLogger.warn("Navigation to start URL blocked", {
+                  targetUrl,
+                  startUrl: state.startUrl,
+                  step: state.step,
+                });
+                // Set navigation blocked for next prompt feedback
+                state.navigationBlocked = {
+                  url: targetUrl,
+                  reason: "Cannot navigate back to start URL mid-flow. Try interacting with elements on the current page instead.",
+                };
+                continue;
+              }
+            }
+
+            // Validate action
+            const actionValidation = securityValidator.validateAction(
+              action,
+              allowedDomains
+            );
+            if (!actionValidation.valid) {
+              iterationLogger.warn("Action blocked by security validator", {
+                action,
+                reason: actionValidation.reason,
+              });
+              continue;
+            }
+
+            // Check for exfiltration
+            const exfilCheck = securityValidator.detectExfiltrationAttempt(
+              action,
+              snapshot.content
+            );
+            if (exfilCheck.detected) {
+              iterationLogger.warn("Exfiltration attempt blocked", {
+                type: exfilCheck.type,
+                evidence: exfilCheck.evidence,
+              });
+              continue;
+            }
+
+            // Execute action
+            const actionKey = `${action.tool}:${JSON.stringify(action.args)}`;
+
+            // Check for action repeat
+            const recentActions = state.loopDetection.recentActions;
+            if (
+              recentActions.length >= 3 &&
+              recentActions.slice(-3).every((a) => a === actionKey)
+            ) {
+              iterationLogger.warn("Action repeat blocked", { action: actionKey });
+              continue;
+            }
+
+            await executeAction(playwrightClient, action, iterationLogger);
+
+            // Record action
+            await workspaceManager.recordAction(input.analysisId, crawlId, {
+              step: state.step,
+              timestamp: new Date().toISOString(),
+              tool: action.tool,
+              args: action.args,
+              result: "success",
+              reasoning: actionResponse.reasoning,
+            });
+
+            state.actionHistory.push(
+              `Step ${state.step}: ${action.tool}(${JSON.stringify(action.args)}) - ${actionResponse.reasoning}`
+            );
+
+            recentActions.push(actionKey);
+            if (recentActions.length > 10) {
+              recentActions.shift();
+            }
+          }
+
+          state.step++;
+
+          // Checkpoint every N steps
+          if (state.step % config.checkpointInterval === 0) {
+            const checkpoint: CrawlCheckpoint = {
+              step: state.step,
+              timestamp: new Date().toISOString(),
+              visitedUrls: Array.from(state.visitedUrls),
+              currentUrl: state.currentUrl,
+              goalProgress: state.goalProgress,
+              canResume: true,
+              startUrl: state.startUrl,
+              // Preserve loop detection state for context continuity
+              loopDetection: {
+                domSignatures: Array.from(state.loopDetection.domSignatures.entries()),
+                urlVisits: Array.from(state.loopDetection.urlVisits.entries()),
+                recentActions: [...state.loopDetection.recentActions],
+              },
+            };
+
+            await workspaceManager.saveCheckpoint(
+              input.analysisId,
+              crawlId,
+              checkpoint
+            );
+
+            iterationLogger.info("Checkpoint saved", { step: state.step });
+          }
+        }
+
+        // Finalize crawl
+        const finalStatus = goalComplete
+          ? "completed"
+          : blocked
+            ? "failed"
+            : "completed";
+
+        await workspaceManager.updateCrawlIndex(input.analysisId, crawlId, {
+          status: finalStatus,
+          completedAt: new Date().toISOString(),
+        });
+
+        // Update workspace crawl reference
+        const updatedWorkspace = await workspaceManager.readWorkspaceIndex(
+          input.analysisId
+        );
+        const crawlRef = updatedWorkspace.crawls.find((c) => c.crawlId === crawlId);
+        if (crawlRef) {
+          crawlRef.status = finalStatus;
+          crawlRef.completedAt = new Date().toISOString();
+          crawlRef.pagesVisited = state.visitedUrls.size;
+          crawlRef.stepsExecuted = state.step;
+          await workspaceManager.updateWorkspaceIndex(input.analysisId, {
+            crawls: updatedWorkspace.crawls,
+          });
+        }
+
+        await resourceManager.notifyListChanged();
+
+        const crawlIndex = await workspaceManager.readCrawlIndex(
+          input.analysisId,
+          crawlId
+        );
+
+        const result = {
+          crawlId,
+          status: finalStatus,
+          goalComplete,
+          blocked,
+          blockedReason,
+          pagesVisited: crawlIndex.pages.length,
+          stepsExecuted: state.step,
+          crawlIndexFilePath: join(crawlPath, "index.md"),
+          crawlIndexUri: `webtest://${input.analysisId}/crawls/${crawlId}/index.md`,
+          summaryUri: `webtest://${input.analysisId}/crawls/${crawlId}/summary.md`,
+          nextSteps: goalComplete
+            ? [
+                `Use webtest_analyze_app with analysisId="${input.analysisId}" to analyze the application`,
+              ]
+            : blocked
+              ? [`Resolve the blocker: ${blockedReason}`]
+              : [
+                  `Crawl completed with partial results`,
+                  `Use webtest_analyze_app to analyze what was discovered`,
+                ],
+        };
+
+        iterationLogger.info("Crawl completed", {
+          status: finalStatus,
+          pagesVisited: result.pagesVisited,
+          stepsExecuted: result.stepsExecuted,
+        });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(result, null, 2),
+            },
+          ],
+        };
+      } catch (error) {
+        if (error instanceof CancellationError) {
+          crawlLogger.info("Crawl cancelled", { requestId: error.requestId });
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify(
+                  {
+                    status: "cancelled",
+                    message: "Crawl was cancelled by user",
+                    partialResultsAvailable: true,
+                  },
+                  null,
+                  2
+                ),
+              },
+            ],
+          };
+        }
+
+        const message = error instanceof Error ? error.message : "Unknown error";
+        crawlLogger.error("Crawl failed", { error: message });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Error during crawl: ${message}`,
+            },
+          ],
+          isError: true,
+        };
+      } finally {
+        cancellationRegistry.unregister(requestId);
+      }
+    },
+  };
+}
+
+function createInitialState(startUrl: string): CrawlState {
+  return {
+    step: 0,
+    currentUrl: startUrl,
+    startUrl,
+    visitedUrls: new Set(),
+    actionHistory: [],
+    goalProgress: "Starting exploration",
+    loopDetection: {
+      domSignatures: new Map(),
+      urlVisits: new Map(),
+      recentActions: [],
+    },
+  };
+}
+
+/**
+ * Check if a target URL is the start URL (for navigation guard).
+ * Compares normalized URLs to handle variations like trailing slashes and default paths.
+ */
+function isStartUrlNavigation(targetUrl: string, startUrl: string): boolean {
+  try {
+    const target = new URL(targetUrl, startUrl);
+    const start = new URL(startUrl);
+
+    // Normalize paths (remove trailing slash, handle "/" as empty)
+    const normalizePathname = (p: string) => {
+      const normalized = p.replace(/\/$/, "") || "/";
+      return normalized;
+    };
+
+    // Compare origin and pathname
+    const targetPath = normalizePathname(target.pathname);
+    const startPath = normalizePathname(start.pathname);
+
+    return target.origin === start.origin && targetPath === startPath;
+  } catch {
+    // If URL parsing fails, do simple string comparison
+    return targetUrl === startUrl || targetUrl === "/" || targetUrl === "";
+  }
+}
+
+async function executeAction(
+  playwright: PlaywrightClient,
+  action: { tool: string; args: Record<string, unknown> },
+  logger: { debug: (msg: string, data?: Record<string, unknown>) => void }
+): Promise<void> {
+  logger.debug("Executing action", { tool: action.tool, args: action.args });
+
+  switch (action.tool) {
+    case "navigate":
+      await playwright.navigate(action.args.url as string);
+      break;
+    case "click":
+      await playwright.click(
+        action.args.element as string,
+        action.args.ref as string
+      );
+      break;
+    case "type":
+      await playwright.type(
+        action.args.element as string,
+        action.args.ref as string,
+        action.args.text as string,
+        {
+          submit: action.args.submit as boolean | undefined,
+          slowly: action.args.slowly as boolean | undefined,
+        }
+      );
+      break;
+    case "fill":
+      await playwright.fill(
+        action.args.element as string,
+        action.args.ref as string,
+        action.args.value as string
+      );
+      break;
+    case "hover":
+      await playwright.hover(
+        action.args.element as string,
+        action.args.ref as string
+      );
+      break;
+    case "select":
+      await playwright.select(
+        action.args.element as string,
+        action.args.ref as string,
+        action.args.values as string[]
+      );
+      break;
+    case "press":
+      await playwright.press(action.args.key as string);
+      break;
+    case "scroll":
+      await playwright.scroll(
+        action.args.x as number,
+        action.args.y as number
+      );
+      break;
+    case "wait":
+      await playwright.wait(action.args.ms as number);
+      break;
+    default:
+      logger.debug("Unknown action tool", { tool: action.tool });
+  }
+}
+
+async function handleElicitation(
+  elicitationClient: ElicitationClient,
+  elicitation: NonNullable<CrawlAction["elicitationNeeded"]>,
+  logger: { info: (msg: string, data?: Record<string, unknown>) => void }
+): Promise<{ selectedValue?: string; cancelled?: boolean }> {
+  logger.info("Handling elicitation", {
+    type: elicitation.type,
+    context: elicitation.context,
+  });
+
+  let request;
+  switch (elicitation.type) {
+    case "cookie_consent":
+      request = elicitationClient.createCookieConsentRequest(elicitation.context);
+      break;
+    case "modal_blocking":
+      request = elicitationClient.createModalBlockingRequest(elicitation.context);
+      break;
+    case "ambiguous_navigation":
+      request = elicitationClient.createAmbiguousNavigationRequest(
+        (elicitation.options || []).map(opt => ({
+          url: opt.url || "",
+          label: opt.label,
+        }))
+      );
+      break;
+    case "auth_required":
+      request = elicitationClient.createAuthRequiredRequest();
+      break;
+    default:
+      return {};
+  }
+
+  const result = await elicitationClient.elicit(request);
+
+  if (!result.success) {
+    // Fallback: include questions in next sampling prompt
+    logger.info("Elicitation fallback", {
+      fallbackQuestions: result.fallbackQuestions,
+    });
+    return {};
+  }
+
+  return {
+    selectedValue: result.selectedValue,
+    cancelled: result.cancelled,
+  };
+}