retestkit 1.4.1

package/src/tools/webtest/discover-features.ts

@@ -0,0 +1,412 @@
+import { z } from "zod";
+import type { McpTool, ToolResult } from "../../types/tool.js";
+import type { ServerContext } from "../../types/context.js";
+import type { WorkspaceManager, Feature } from "../../workspace/index.js";
+import type { SamplingClient } from "../../sampling/index.js";
+import type { ResourceManager } from "../../resources/index.js";
+import { buildFeatureDiscoveryPrompt } from "../../sampling/prompts.js";
+import { FeaturesDiscoverySchema, AnalysisIdSchema, CrawlIdSchema, type FeaturesDiscovery } from "./schemas.js";
+
+export const discoverFeaturesInputSchema = z.object({
+  analysisId: AnalysisIdSchema,
+  crawlId: CrawlIdSchema
+    .optional()
+    .describe("Specific crawl to analyze. If not provided, uses the most recent crawl with captured pages."),
+  manualDiscovery: z
+    .object({
+      appPurpose: z.string(),
+      appType: z.string(),
+      features: z.array(z.object({
+        slug: z.string(),
+        name: z.string(),
+        description: z.string(),
+        entities: z.array(z.string()),
+        entryPoints: z.array(z.string()),
+      })),
+      securityObservations: z.array(z.string()).optional(),
+      accessibilityObservations: z.array(z.string()).optional(),
+    })
+    .optional()
+    .describe("Manual discovery result when sampling is unavailable. Provide this after executing the returned prompt."),
+});
+
+export type DiscoverFeaturesInput = z.infer<typeof discoverFeaturesInputSchema>;
+
+export function createDiscoverFeaturesTool(
+  getContext: () => ServerContext & {
+    workspaceManager: WorkspaceManager;
+    samplingClient: SamplingClient;
+    resourceManager: ResourceManager;
+  }
+): McpTool<DiscoverFeaturesInput> {
+  return {
+    name: "webtest_discover_features",
+    description: `Discover application features/modules from crawl data.
+
+This tool uses AI to identify distinct features of the web application:
+- Identifies the application purpose and type
+- Discovers distinct features/modules (e.g., Authentication, Shopping Cart, Search)
+- Lists entities and entry points for each feature
+- Notes security and accessibility observations
+
+Outputs features.md at workspace root. After discovering features, use webtest_discover_flows to identify user flows within each feature.
+
+Requires a crawl with captured pages. If sampling is unavailable, returns a prompt for manual execution.`,
+
+    inputSchema: discoverFeaturesInputSchema,
+
+    async handler(input: DiscoverFeaturesInput): Promise<ToolResult> {
+      const ctx = getContext();
+      const { logger, workspaceManager, samplingClient, resourceManager } = ctx;
+
+      const discoveryLogger = logger.withCorrelation({
+        analysisId: input.analysisId,
+      });
+
+      discoveryLogger.info("Starting feature discovery", { crawlId: input.crawlId });
+
+      try {
+        // Validate workspace exists
+        if (!(await workspaceManager.workspaceExists(input.analysisId))) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Error: Analysis workspace "${input.analysisId}" not found.`,
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        const workspace = await workspaceManager.readWorkspaceIndex(input.analysisId);
+
+        // Find the crawl to analyze
+        let targetCrawlId = input.crawlId;
+        let selectedCrawlRef: (typeof workspace.crawls)[0] | undefined;
+
+        if (!targetCrawlId) {
+          // Sort crawls by most recent first
+          const sortedCrawls = [...workspace.crawls].sort(
+            (a, b) =>
+              new Date(b.completedAt || b.startedAt).getTime() -
+              new Date(a.completedAt || a.startedAt).getTime()
+          );
+
+          // First try crawls with pagesVisited > 0 in workspace index
+          const crawlsWithData = sortedCrawls.filter((c) => c.pagesVisited > 0);
+
+          if (crawlsWithData.length > 0) {
+            selectedCrawlRef = crawlsWithData[0];
+            targetCrawlId = selectedCrawlRef.crawlId;
+          } else if (sortedCrawls.length > 0) {
+            // Fallback: check actual crawl indexes for pages (workspace index may be stale)
+            for (const crawlRef of sortedCrawls) {
+              try {
+                const crawlIndex = await workspaceManager.readCrawlIndex(
+                  input.analysisId,
+                  crawlRef.crawlId
+                );
+                if (crawlIndex.pages.length > 0) {
+                  selectedCrawlRef = crawlRef;
+                  targetCrawlId = crawlRef.crawlId;
+                  discoveryLogger.info("Found crawl with data via index check", {
+                    crawlId: crawlRef.crawlId,
+                    pagesInIndex: crawlIndex.pages.length,
+                    pagesInWorkspace: crawlRef.pagesVisited,
+                  });
+                  break;
+                }
+              } catch {
+                // Crawl index not readable, skip
+              }
+            }
+          }
+
+          if (!targetCrawlId) {
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: "Error: No crawls with captured data found. Run webtest_crawl_app first.",
+                },
+              ],
+              isError: true,
+            };
+          }
+        } else {
+          selectedCrawlRef = workspace.crawls.find((c) => c.crawlId === targetCrawlId);
+        }
+
+        // Load crawl data
+        const crawlIndex = await workspaceManager.readCrawlIndex(
+          input.analysisId,
+          targetCrawlId
+        );
+
+        if (crawlIndex.pages.length === 0) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "Error: Crawl has no captured pages. The crawl may have failed.",
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        discoveryLogger.info("Analyzing crawl data for features", {
+          crawlId: targetCrawlId,
+          pageCount: crawlIndex.pages.length,
+        });
+
+        // Build crawl summary
+        const crawlSummary = `
+Target URL: ${workspace.url}
+Domain: ${workspace.domain}
+Focus: ${workspace.focus || "General exploration"}
+Goal: ${crawlIndex.goal}
+Strategy: ${crawlIndex.strategy}
+Pages visited: ${crawlIndex.pages.length}
+Actions taken: ${crawlIndex.actionHistory.length}
+Status: ${crawlIndex.status}
+`;
+
+        // Load page snapshots (limited to avoid token limits)
+        const pageSnapshots: Array<{ url: string; content: string }> = [];
+
+        for (const page of crawlIndex.pages.slice(0, 10)) {
+          try {
+            const snapshotContent = await resourceManager.readResource(
+              page.snapshotUri
+            );
+            if (snapshotContent.text) {
+              const snapshot = JSON.parse(snapshotContent.text);
+              pageSnapshots.push({
+                url: page.url,
+                content: snapshot.content || "",
+              });
+            }
+          } catch (error) {
+            discoveryLogger.warn("Failed to load page snapshot", {
+              pageId: page.pageId,
+              error: error instanceof Error ? error.message : "Unknown error",
+            });
+          }
+        }
+
+        // Determine discovery source: manual input, sampling, or fallback prompt
+        let discovery: FeaturesDiscovery;
+
+        if (input.manualDiscovery) {
+          // Use manually provided discovery (from fallback mode)
+          discoveryLogger.info("Using manual discovery input");
+          discovery = input.manualDiscovery as FeaturesDiscovery;
+        } else if (!samplingClient.hasSampling()) {
+          // No sampling available and no manual input - return prompt for manual execution
+          const prompt = buildFeatureDiscoveryPrompt({
+            crawlSummary,
+            pageSnapshots,
+          });
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify(
+                  {
+                    needsManualInput: true,
+                    analysisId: input.analysisId,
+                    crawlId: targetCrawlId,
+                    prompt,
+                    expectedResponseSchema: FeaturesDiscoverySchema._def,
+                    instructions:
+                      "Execute this prompt with your LLM, then call webtest_discover_features again with the result in the 'manualDiscovery' parameter",
+                  },
+                  null,
+                  2
+                ),
+              },
+            ],
+          };
+        } else {
+          // Request discovery via sampling
+          const samplingResult = await samplingClient.createMessage({
+            systemPrompt: `You are analyzing a web application to identify its distinct features and modules.
+Each feature should be a cohesive capability that users interact with (e.g., Authentication, Shopping Cart, Search).
+Provide thorough feature identification that will help in discovering user flows within each feature.`,
+            userPrompt: buildFeatureDiscoveryPrompt({
+              crawlSummary,
+              pageSnapshots,
+            }),
+            schema: FeaturesDiscoverySchema,
+            maxTokens: 4096,
+          });
+
+          if (!samplingResult.success || !samplingResult.data) {
+            discoveryLogger.error("Feature discovery sampling failed", {
+              error: samplingResult.error,
+            });
+
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: `Error during feature discovery: ${samplingResult.error}`,
+              },
+            ],
+            isError: true,
+            };
+          }
+
+          discovery = samplingResult.data;
+        }
+
+        // Generate markdown report
+        const markdownReport = generateFeaturesMarkdown(discovery, workspace, crawlIndex);
+
+        // Build frontmatter with features
+        const frontmatter = {
+          appPurpose: discovery.appPurpose,
+          appType: discovery.appType,
+          features: discovery.features,
+          securityObservations: discovery.securityObservations,
+          accessibilityObservations: discovery.accessibilityObservations,
+        };
+
+        // Save features
+        const { featuresFilePath, featuresUri, featureCount } =
+          await workspaceManager.saveFeatures(input.analysisId, {
+            markdown: markdownReport,
+            frontmatter: frontmatter as { features: Feature[] },
+          });
+
+        await resourceManager.notifyListChanged();
+
+        // Build warning if crawl was incomplete
+        const crawlStatus = selectedCrawlRef?.status;
+        const isIncompleteCrawl = crawlStatus && crawlStatus !== "completed";
+        const warning = isIncompleteCrawl
+          ? `Note: Discovery based on incomplete crawl (status: ${crawlStatus}, pages: ${selectedCrawlRef?.pagesVisited || crawlIndex.pages.length}). Results may be partial.`
+          : undefined;
+
+        // Build next steps suggesting to discover flows for each feature
+        const nextSteps = discovery.features.map(
+          (f) => `Use webtest_discover_flows with analysisId="${input.analysisId}" featureSlug="${f.slug}" to discover flows for "${f.name}"`
+        );
+        nextSteps.push(`Review discovered features at ${featuresUri}`);
+
+        const result = {
+          analysisId: input.analysisId,
+          crawlId: targetCrawlId,
+          ...(warning && { warning }),
+          featuresFilePath,
+          featuresUri,
+          summary: {
+            appPurpose: discovery.appPurpose,
+            appType: discovery.appType,
+            featureCount,
+            features: discovery.features.map((f) => ({
+              slug: f.slug,
+              name: f.name,
+            })),
+            crawlStatus: crawlStatus || "unknown",
+          },
+          nextSteps,
+        };
+
+        discoveryLogger.info("Feature discovery completed", {
+          featureCount,
+          features: discovery.features.map((f) => f.slug),
+        });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(result, null, 2),
+            },
+          ],
+        };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : "Unknown error";
+        discoveryLogger.error("Feature discovery failed", { error: message });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Error during feature discovery: ${message}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  };
+}
+
+function generateFeaturesMarkdown(
+  discovery: FeaturesDiscovery,
+  workspace: { url: string; domain: string; focus?: string },
+  crawl: { goal: string; pages: { url: string }[] }
+): string {
+  let md = `# Application Features: ${workspace.domain}
+
+## Overview
+
+- **URL**: ${workspace.url}
+- **Purpose**: ${discovery.appPurpose}
+- **Type**: ${discovery.appType}
+- **Focus**: ${workspace.focus || "General"}
+- **Crawl Goal**: ${crawl.goal}
+- **Pages Analyzed**: ${crawl.pages.length}
+
+## Discovered Features
+
+`;
+
+  for (const feature of discovery.features) {
+    md += `### ${feature.name}
+
+**Slug**: \`${feature.slug}\`
+
+${feature.description}
+
+**Entities**: ${feature.entities.length > 0 ? feature.entities.join(", ") : "None identified"}
+
+**Entry Points**:
+`;
+    for (const ep of feature.entryPoints) {
+      md += `- ${ep}\n`;
+    }
+    md += "\n";
+  }
+
+  if (discovery.securityObservations && discovery.securityObservations.length > 0) {
+    md += `## Security Observations
+
+`;
+    for (const obs of discovery.securityObservations) {
+      md += `- ${obs}\n`;
+    }
+    md += "\n";
+  }
+
+  if (discovery.accessibilityObservations && discovery.accessibilityObservations.length > 0) {
+    md += `## Accessibility Observations
+
+`;
+    for (const obs of discovery.accessibilityObservations) {
+      md += `- ${obs}\n`;
+    }
+    md += "\n";
+  }
+
+  md += `---
+*Generated by testing-mcp*
+`;
+
+  return md;
+}