retestkit 1.4.1

package/src/tools/webtest/discover-flows.ts

@@ -0,0 +1,408 @@
+import { z } from "zod";
+import type { McpTool, ToolResult } from "../../types/tool.js";
+import type { ServerContext } from "../../types/context.js";
+import type { WorkspaceManager } from "../../workspace/index.js";
+import type { SamplingClient } from "../../sampling/index.js";
+import type { ResourceManager } from "../../resources/index.js";
+import { buildFlowDiscoveryPrompt } from "../../sampling/prompts.js";
+import { FlowsDiscoverySchema, AnalysisIdSchema, CrawlIdSchema, FeatureSlugSchema, type FlowsDiscovery } from "./schemas.js";
+
+export const discoverFlowsInputSchema = z.object({
+  analysisId: AnalysisIdSchema,
+  featureSlug: FeatureSlugSchema,
+  crawlId: CrawlIdSchema
+    .optional()
+    .describe("Specific crawl to use. If not provided, uses the most recent crawl with captured pages."),
+  manualDiscovery: z
+    .object({
+      featureSlug: z.string(),
+      flows: z.array(z.object({
+        id: z.string(),
+        name: z.string(),
+        description: z.string(),
+        entryPoint: z.string(),
+        steps: z.array(z.string()),
+      })),
+      suggestedAssertions: z.array(z.string()),
+    })
+    .optional()
+    .describe("Manual discovery result when sampling is unavailable. Provide this after executing the returned prompt."),
+});
+
+export type DiscoverFlowsInput = z.infer<typeof discoverFlowsInputSchema>;
+
+export function createDiscoverFlowsTool(
+  getContext: () => ServerContext & {
+    workspaceManager: WorkspaceManager;
+    samplingClient: SamplingClient;
+    resourceManager: ResourceManager;
+  }
+): McpTool<DiscoverFlowsInput> {
+  return {
+    name: "webtest_discover_flows",
+    description: `Discover user flows within a specific feature.
+
+This tool uses AI to identify user flows within a feature:
+- Identifies user journeys that accomplish specific goals
+- Maps entry points and step sequences for each flow
+- Suggests assertions that should hold for the flows
+
+Outputs flows.md under features/<feature-slug>/. Run webtest_discover_features first to identify features.
+
+Requires a crawl with captured pages and a valid feature slug from features.md.`,
+
+    inputSchema: discoverFlowsInputSchema,
+
+    async handler(input: DiscoverFlowsInput): Promise<ToolResult> {
+      const ctx = getContext();
+      const { logger, workspaceManager, samplingClient, resourceManager } = ctx;
+
+      const discoveryLogger = logger.withCorrelation({
+        analysisId: input.analysisId,
+        featureSlug: input.featureSlug,
+      });
+
+      discoveryLogger.info("Starting flow discovery", {
+        featureSlug: input.featureSlug,
+        crawlId: input.crawlId,
+      });
+
+      try {
+        // Validate workspace exists
+        if (!(await workspaceManager.workspaceExists(input.analysisId))) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Error: Analysis workspace "${input.analysisId}" not found.`,
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        const workspace = await workspaceManager.readWorkspaceIndex(input.analysisId);
+
+        // Check if features have been discovered
+        if (!workspace.features) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "Error: No features found. Run webtest_discover_features first.",
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        // Find the requested feature
+        const feature = workspace.features.features.find(
+          (f) => f.slug === input.featureSlug
+        );
+
+        if (!feature) {
+          const availableFeatures = workspace.features.features.map((f) => f.slug).join(", ");
+          return {
+            content: [
+              {
+                type: "text",
+                text: `Error: Feature "${input.featureSlug}" not found. Available features: ${availableFeatures}`,
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        // Find the crawl to use
+        let targetCrawlId = input.crawlId;
+        let selectedCrawlRef: (typeof workspace.crawls)[0] | undefined;
+
+        if (!targetCrawlId) {
+          // Sort crawls by most recent first
+          const sortedCrawls = [...workspace.crawls].sort(
+            (a, b) =>
+              new Date(b.completedAt || b.startedAt).getTime() -
+              new Date(a.completedAt || a.startedAt).getTime()
+          );
+
+          const crawlsWithData = sortedCrawls.filter((c) => c.pagesVisited > 0);
+
+          if (crawlsWithData.length > 0) {
+            selectedCrawlRef = crawlsWithData[0];
+            targetCrawlId = selectedCrawlRef.crawlId;
+          } else if (sortedCrawls.length > 0) {
+            for (const crawlRef of sortedCrawls) {
+              try {
+                const crawlIndex = await workspaceManager.readCrawlIndex(
+                  input.analysisId,
+                  crawlRef.crawlId
+                );
+                if (crawlIndex.pages.length > 0) {
+                  selectedCrawlRef = crawlRef;
+                  targetCrawlId = crawlRef.crawlId;
+                  break;
+                }
+              } catch {
+                // Crawl index not readable, skip
+              }
+            }
+          }
+
+          if (!targetCrawlId) {
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: "Error: No crawls with captured data found.",
+                },
+              ],
+              isError: true,
+            };
+          }
+        } else {
+          selectedCrawlRef = workspace.crawls.find((c) => c.crawlId === targetCrawlId);
+        }
+
+        // Load crawl data
+        const crawlIndex = await workspaceManager.readCrawlIndex(
+          input.analysisId,
+          targetCrawlId
+        );
+
+        if (crawlIndex.pages.length === 0) {
+          return {
+            content: [
+              {
+                type: "text",
+                text: "Error: Crawl has no captured pages.",
+              },
+            ],
+            isError: true,
+          };
+        }
+
+        discoveryLogger.info("Discovering flows for feature", {
+          featureName: feature.name,
+          crawlId: targetCrawlId,
+          pageCount: crawlIndex.pages.length,
+        });
+
+        // Filter page snapshots relevant to the feature (by entry points)
+        // For now, load all pages and let the AI filter
+        const pageSnapshots: Array<{ url: string; content: string }> = [];
+
+        for (const page of crawlIndex.pages.slice(0, 10)) {
+          try {
+            const snapshotContent = await resourceManager.readResource(
+              page.snapshotUri
+            );
+            if (snapshotContent.text) {
+              const snapshot = JSON.parse(snapshotContent.text);
+              pageSnapshots.push({
+                url: page.url,
+                content: snapshot.content || "",
+              });
+            }
+          } catch (error) {
+            discoveryLogger.warn("Failed to load page snapshot", {
+              pageId: page.pageId,
+              error: error instanceof Error ? error.message : "Unknown error",
+            });
+          }
+        }
+
+        // Determine discovery source: manual input, sampling, or fallback prompt
+        let discovery: FlowsDiscovery;
+
+        if (input.manualDiscovery) {
+          discoveryLogger.info("Using manual discovery input");
+          discovery = input.manualDiscovery as FlowsDiscovery;
+        } else if (!samplingClient.hasSampling()) {
+          const prompt = buildFlowDiscoveryPrompt({
+            featureSlug: feature.slug,
+            featureName: feature.name,
+            featureDescription: feature.description,
+            featureEntities: feature.entities,
+            featureEntryPoints: feature.entryPoints,
+            pageSnapshots,
+          });
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify(
+                  {
+                    needsManualInput: true,
+                    analysisId: input.analysisId,
+                    featureSlug: input.featureSlug,
+                    crawlId: targetCrawlId,
+                    prompt,
+                    expectedResponseSchema: FlowsDiscoverySchema._def,
+                    instructions:
+                      "Execute this prompt with your LLM, then call webtest_discover_flows again with the result in the 'manualDiscovery' parameter",
+                  },
+                  null,
+                  2
+                ),
+              },
+            ],
+          };
+        } else {
+          const samplingResult = await samplingClient.createMessage({
+            systemPrompt: `You are identifying user flows within the "${feature.name}" feature of a web application.
+A "flow" is a user journey that accomplishes a specific goal within this feature.
+Focus only on flows relevant to this specific feature.`,
+            userPrompt: buildFlowDiscoveryPrompt({
+              featureSlug: feature.slug,
+              featureName: feature.name,
+              featureDescription: feature.description,
+              featureEntities: feature.entities,
+              featureEntryPoints: feature.entryPoints,
+              pageSnapshots,
+            }),
+            schema: FlowsDiscoverySchema,
+            maxTokens: 4096,
+          });
+
+          if (!samplingResult.success || !samplingResult.data) {
+            discoveryLogger.error("Flow discovery sampling failed", {
+              error: samplingResult.error,
+            });
+
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: `Error during flow discovery: ${samplingResult.error}`,
+                },
+              ],
+              isError: true,
+            };
+          }
+
+          discovery = samplingResult.data;
+        }
+
+        // Generate flows markdown
+        const markdownReport = generateFlowsMarkdown(discovery, feature);
+
+        // Build frontmatter
+        const frontmatter = {
+          featureSlug: discovery.featureSlug,
+          featureName: feature.name,
+          flows: discovery.flows,
+          suggestedAssertions: discovery.suggestedAssertions,
+        };
+
+        // Save flows
+        const { flowsFilePath, flowsUri, flowCount } =
+          await workspaceManager.saveFlows(input.analysisId, input.featureSlug, {
+            markdown: markdownReport,
+            frontmatter,
+          });
+
+        await resourceManager.notifyListChanged();
+
+        const result = {
+          analysisId: input.analysisId,
+          featureSlug: input.featureSlug,
+          crawlId: targetCrawlId,
+          flowsFilePath,
+          flowsUri,
+          summary: {
+            featureName: feature.name,
+            flowCount,
+            flows: discovery.flows.map((f) => ({
+              id: f.id,
+              name: f.name,
+            })),
+            assertionCount: discovery.suggestedAssertions.length,
+          },
+          nextSteps: [
+            `Use webtest_generate_tests with analysisId="${input.analysisId}" to generate test cases`,
+            `Review discovered flows at ${flowsUri}`,
+          ],
+        };
+
+        discoveryLogger.info("Flow discovery completed", {
+          featureSlug: input.featureSlug,
+          flowCount,
+          flows: discovery.flows.map((f) => f.id),
+        });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(result, null, 2),
+            },
+          ],
+        };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : "Unknown error";
+        discoveryLogger.error("Flow discovery failed", { error: message });
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Error during flow discovery: ${message}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  };
+}
+
+function generateFlowsMarkdown(
+  discovery: FlowsDiscovery,
+  feature: { name: string; slug: string; description: string }
+): string {
+  let md = `# User Flows: ${feature.name}
+
+## Feature Overview
+
+**Slug**: \`${feature.slug}\`
+
+${feature.description}
+
+## Flows
+
+`;
+
+  for (const flow of discovery.flows) {
+    md += `### ${flow.name}
+
+**ID**: \`${flow.id}\`
+**Entry Point**: ${flow.entryPoint}
+
+${flow.description}
+
+**Steps**:
+`;
+    for (let i = 0; i < flow.steps.length; i++) {
+      md += `${i + 1}. ${flow.steps[i]}\n`;
+    }
+    md += "\n---\n\n";
+  }
+
+  if (discovery.suggestedAssertions.length > 0) {
+    md += `## Suggested Assertions
+
+`;
+    for (const assertion of discovery.suggestedAssertions) {
+      md += `- ${assertion}\n`;
+    }
+    md += "\n";
+  }
+
+  md += `---
+*Generated by testing-mcp*
+`;
+
+  return md;
+}