web-tester-for-claude 0.4.0

package/src/map/crawl.ts

@@ -0,0 +1,394 @@
+import { resolve } from "node:path";
+import { chromium, type Browser, type BrowserContext } from "playwright";
+import { readUiAttributes } from "../browser/attrs";
+import {
+  configureContext,
+  DEFAULT_SESSION_UA,
+  DEFAULT_SESSION_VIEWPORT
+} from "../browser/session";
+import { attachCapture } from "../inspector/capture";
+import { log } from "../util/log";
+import { SESSION_STATE_PATH } from "../util/paths";
+import { existsSync } from "node:fs";
+import { routeTemplate } from "./classify";
+
+/** A form discovered on a page, with enough detail to draft a journey. */
+export type PageForm = {
+  action: string | null;
+  method: string;
+  fields: { name: string; type: string; tag: string; required: boolean }[];
+  submitText: string | null;
+};
+
+/** Everything the crawler observes about one page. */
+export type PageFacts = {
+  /** Absolute URL requested. */
+  url: string;
+  /** Path requested (pathname + search). */
+  path: string;
+  /** Path after any redirects. */
+  finalPath: string;
+  status: number | null;
+  title: string;
+  ok: boolean;
+  depth: number;
+  hasHeader: boolean;
+  hasFooter: boolean;
+  hasMain: boolean;
+  hasNav: boolean;
+  h1: string | null;
+  headingCount: number;
+  forms: PageForm[];
+  /** Same-origin paths linked from this page. */
+  internalLinks: string[];
+  imageCount: number;
+  /** `main a[href^='/']:has(img)` — catalog-card shape. */
+  cardLinkCount: number;
+  passwordFields: number;
+  searchInputs: number;
+  consoleErrors: number;
+  pageErrors: number;
+  attrCount: number;
+  /** Relative path to a viewport screenshot under the map dir, if captured. */
+  screenshot?: string;
+  error?: string;
+};
+
+export type CrawlOptions = {
+  baseUrl: string;
+  /** Seed paths to start from (the base path + any sitemap entries). */
+  seeds: string[];
+  mapDir: string;
+  limit: number;
+  depth: number;
+  concurrency: number;
+  perTemplate: number;
+  captureScreenshots: boolean;
+  loadStorageState: boolean;
+  gotoTimeoutMs: number;
+  filter?: RegExp;
+  exclude?: RegExp;
+};
+
+type DomFacts = Omit<
+  PageFacts,
+  | "url"
+  | "path"
+  | "finalPath"
+  | "status"
+  | "depth"
+  | "ok"
+  | "consoleErrors"
+  | "pageErrors"
+  | "attrCount"
+  | "screenshot"
+  | "error"
+>;
+
+/**
+ * Run inside the page to collect structural facts in one round-trip.
+ *
+ * Everything in here is serialised to the browser by page.evaluate, so it must
+ * use inline callback arguments only — no `const fn = () => …` and no nested
+ * `function` declarations. tsx's esbuild "keep names" transform wraps named
+ * inner functions in a `__name(…)` helper that doesn't exist in the page; one
+ * slips in and the whole evaluate throws `ReferenceError: __name is not
+ * defined`, which the caller swallows — leaving every fact blank.
+ */
+function collectDomFacts(): DomFacts {
+  const forms = Array.from(document.querySelectorAll("form")).map((form) => {
+    const fields = Array.from(
+      form.querySelectorAll("input, select, textarea")
+    )
+      .map((el) => {
+        const tag = el.tagName.toLowerCase();
+        const type =
+          tag === "input"
+            ? (el.getAttribute("type") ?? "text").toLowerCase()
+            : tag;
+        return {
+          name:
+            el.getAttribute("name") ?? el.getAttribute("id") ?? "",
+          type,
+          tag,
+          required: el.hasAttribute("required")
+        };
+      })
+      .filter((f) => f.type !== "hidden");
+    const submit = form.querySelector(
+      "button[type=submit], input[type=submit], button:not([type])"
+    );
+    return {
+      action: form.getAttribute("action"),
+      method: (form.getAttribute("method") ?? "get").toLowerCase(),
+      fields,
+      submitText: submit
+        ? (submit.textContent ?? "").replace(/\s+/g, " ").trim() ||
+          submit.getAttribute("value")
+        : null
+    };
+  });
+
+  // `.href` (not getAttribute) gives the browser-resolved absolute URL against
+  // the live document base — so relative links resolve correctly even after a
+  // redirect, where the requested URL and final URL differ.
+  const anchors = Array.from(document.querySelectorAll("a[href]"))
+    .map((a) => (a as HTMLAnchorElement).href)
+    .filter(Boolean);
+
+  return {
+    title: document.title ?? "",
+    hasHeader: !!document.querySelector("header"),
+    hasFooter: !!document.querySelector("footer"),
+    hasMain: !!document.querySelector("main"),
+    hasNav: !!document.querySelector("nav"),
+    h1:
+      (document.querySelector("h1")?.textContent ?? "")
+        .replace(/\s+/g, " ")
+        .trim() || null,
+    headingCount: document.querySelectorAll("h1, h2, h3").length,
+    forms,
+    internalLinks: anchors,
+    imageCount: document.querySelectorAll("img").length,
+    cardLinkCount: document.querySelectorAll("main a[href^='/']:has(img)")
+      .length,
+    passwordFields: document.querySelectorAll("input[type=password]").length,
+    searchInputs: document.querySelectorAll(
+      "input[type=search], input[name=q], input[name=query], input[name=s]"
+    ).length
+  };
+}
+
+/** Normalise an href to a same-origin path (pathname + search), or null. */
+function toInternalPath(href: string, pageUrl: string, origin: string): string | null {
+  const trimmed = href.trim();
+  if (
+    !trimmed ||
+    trimmed.startsWith("#") ||
+    /^(mailto:|tel:|javascript:|data:)/i.test(trimmed)
+  )
+    return null;
+  let abs: URL;
+  try {
+    abs = new URL(trimmed, pageUrl);
+  } catch {
+    return null;
+  }
+  if (abs.origin !== origin) return null;
+  if (!/^https?:$/.test(abs.protocol)) return null;
+  // Drop the hash — fragment-only differences are the same document.
+  return abs.pathname + abs.search;
+}
+
+function slug(path: string, index: number): string {
+  const cleaned = path
+    .replace(/[^a-z0-9]+/gi, "-")
+    .replace(/^-+|-+$/g, "")
+    .toLowerCase()
+    .slice(0, 50);
+  return `${String(index).padStart(3, "0")}-${cleaned || "root"}`;
+}
+
+/**
+ * Breadth-first crawl from the seed paths, same-origin only. Stops at
+ * `limit` fetched pages or `depth` link hops, and fetches at most
+ * `perTemplate` pages per dynamic-route template so a catalog of thousands
+ * of detail pages doesn't dominate the crawl.
+ */
+export async function crawlSite(opts: CrawlOptions): Promise<PageFacts[]> {
+  const origin = new URL(opts.baseUrl).origin;
+  const browser: Browser = await chromium.launch({ headless: true });
+  const useStorageState =
+    opts.loadStorageState && existsSync(SESSION_STATE_PATH);
+  if (useStorageState) log.dim("  · loaded session from ~/.web-tester/session.json");
+
+  const results: PageFacts[] = [];
+  const queued = new Set<string>();
+  const templateCounts = new Map<string, number>();
+  type Job = { path: string; depth: number };
+  const frontier: Job[] = [];
+
+  const wanted = (path: string): boolean => {
+    if (opts.filter && !opts.filter.test(path)) return false;
+    if (opts.exclude && opts.exclude.test(path)) return false;
+    return true;
+  };
+
+  for (const seed of opts.seeds) {
+    if (!queued.has(seed) && wanted(seed)) {
+      queued.add(seed);
+      frontier.push({ path: seed, depth: 0 });
+    }
+  }
+
+  let fetched = 0;
+  let active = 0;
+
+  const basePath = (() => {
+    try {
+      const u = new URL(opts.baseUrl);
+      return (u.pathname.replace(/\/+$/, "") || "/") + u.search;
+    } catch {
+      return "/";
+    }
+  })();
+
+  const claimTemplateSlot = (path: string): boolean => {
+    // The base path always gets crawled. Everything else — including sitemap
+    // seeds — is capped per route template so a large catalog can't crowd
+    // out the rest of the site.
+    if (path === basePath || path === "/") return true;
+    const tpl = routeTemplate(path);
+    const count = templateCounts.get(tpl) ?? 0;
+    if (count >= opts.perTemplate) return false;
+    templateCounts.set(tpl, count + 1);
+    return true;
+  };
+
+  const fetchOne = async (
+    context: BrowserContext,
+    job: Job
+  ): Promise<void> => {
+    const index = ++fetched;
+    const page = await context.newPage();
+    const buffers = attachCapture(context, page, {
+      allNetwork: false,
+      allConsole: false
+    });
+    const url = job.path.startsWith("http")
+      ? job.path
+      : new URL(job.path, opts.baseUrl).toString();
+    let facts: PageFacts = {
+      url,
+      path: job.path,
+      finalPath: job.path,
+      status: null,
+      title: "",
+      ok: false,
+      depth: job.depth,
+      hasHeader: false,
+      hasFooter: false,
+      hasMain: false,
+      hasNav: false,
+      h1: null,
+      headingCount: 0,
+      forms: [],
+      internalLinks: [],
+      imageCount: 0,
+      cardLinkCount: 0,
+      passwordFields: 0,
+      searchInputs: 0,
+      consoleErrors: 0,
+      pageErrors: 0,
+      attrCount: 0
+    };
+    try {
+      const response = await page
+        .goto(url, { waitUntil: "domcontentloaded", timeout: opts.gotoTimeoutMs })
+        .catch((err) => {
+          facts.error = err instanceof Error ? err.message : String(err);
+          return null;
+        });
+      facts.status = response?.status() ?? null;
+      facts.finalPath = (() => {
+        try {
+          const u = new URL(page.url());
+          return u.pathname + u.search;
+        } catch {
+          return job.path;
+        }
+      })();
+      await page.waitForLoadState("load", { timeout: 5_000 }).catch(() => {});
+      const dom = await page.evaluate(collectDomFacts).catch(() => null);
+      if (dom) facts = { ...facts, ...dom };
+      const attrs = await readUiAttributes(page).catch(() => []);
+      facts.attrCount = attrs.length;
+      if (opts.captureScreenshots) {
+        const rel = `${slug(facts.finalPath, index)}.png`;
+        await page
+          .screenshot({ path: resolve(opts.mapDir, rel), fullPage: false })
+          .then(() => {
+            facts.screenshot = rel;
+          })
+          .catch(() => {});
+      }
+    } finally {
+      facts.consoleErrors = buffers.consoleEntries.filter(
+        (e) => e.type === "error"
+      ).length;
+      facts.pageErrors = buffers.pageErrors.length;
+      await page.close().catch(() => {});
+    }
+
+    facts.ok =
+      facts.error === undefined &&
+      facts.status !== null &&
+      facts.status < 400;
+
+    // Resolve and enqueue newly discovered links.
+    const discovered = new Set<string>();
+    for (const href of facts.internalLinks) {
+      const path = toInternalPath(href, facts.url, origin);
+      if (path) discovered.add(path);
+    }
+    facts.internalLinks = Array.from(discovered);
+    if (job.depth < opts.depth) {
+      for (const path of facts.internalLinks) {
+        if (queued.has(path) || !wanted(path)) continue;
+        queued.add(path);
+        frontier.push({ path, depth: job.depth + 1 });
+      }
+    }
+
+    results.push(facts);
+    const tag = facts.ok ? "✓" : "✗";
+    const colour = facts.ok ? log.dim : log.fail;
+    colour(
+      `  ${tag} [${results.length}] ${facts.path} → ${facts.status ?? "?"}${
+        facts.error ? ` (${facts.error.split("\n")[0]})` : ""
+      }`
+    );
+  };
+
+  // Pull from the shared frontier with a fixed worker count. Each worker
+  // keeps its own context. The frontier grows as pages reveal links, so the
+  // loop continues until it drains or the fetch limit is hit.
+  const worker = async (): Promise<void> => {
+    const context = await browser.newContext({
+      viewport: DEFAULT_SESSION_VIEWPORT,
+      userAgent: DEFAULT_SESSION_UA,
+      ...(useStorageState ? { storageState: SESSION_STATE_PATH } : {})
+    });
+    await configureContext(context, opts.baseUrl);
+    try {
+      for (;;) {
+        const job = frontier.shift();
+        if (!job) {
+          if (active === 0) break;
+          await new Promise((r) => setTimeout(r, 25));
+          continue;
+        }
+        if (fetched >= opts.limit) break;
+        if (!claimTemplateSlot(job.path)) continue;
+        active++;
+        try {
+          await fetchOne(context, job);
+        } finally {
+          active--;
+        }
+      }
+    } finally {
+      await context.close().catch(() => {});
+    }
+  };
+
+  try {
+    await Promise.all(
+      Array.from({ length: Math.max(1, opts.concurrency) }, () => worker())
+    );
+  } finally {
+    await browser.close().catch(() => {});
+  }
+
+  return results;
+}

package/src/map/generate.ts

@@ -0,0 +1,253 @@
+import type { ClassifiedPage, RouteGroup } from "./classify";
+import type { PageForm } from "./crawl";
+
+/**
+ * Pick the strongest built-in expectation pack the page actually satisfied
+ * during the crawl, so the generated preset passes on a healthy baseline and
+ * only fails when something regresses. Returns null when nothing fits.
+ */
+export function packForFacts(page: ClassifiedPage): string | null {
+  if (page.type === "home" && page.hasHeader && page.hasFooter)
+    return "homepage";
+  if (page.cardLinkCount >= 1) return "category";
+  if (page.hasMain) return "has-main";
+  if (page.h1) return "has-h1";
+  if (page.hasHeader && page.hasFooter) return "static";
+  return null;
+}
+
+/** Build the `urls-map.txt` preset body from the healthy route groups. */
+export function buildPreset(groups: RouteGroup[], baseUrl: string): string {
+  const lines: string[] = [
+    "# urls-map.txt — generated by `web-tester map`.",
+    "#",
+    `# Source: ${baseUrl}`,
+    "# One representative path per route template, annotated with the",
+    "# strongest expectation pack the page satisfied when crawled. Review and",
+    "# prune, then run:  web-tester sweep --preset map --fail-on http-5xx",
+    ""
+  ];
+  for (const g of groups) {
+    if (g.type === "error" || !g.representative.ok) continue;
+    const pack = packForFacts(g.representative);
+    const path = g.representative.finalPath || g.representative.path;
+    const annotation = pack ? `  #pack=${pack}` : "";
+    const note =
+      g.count > 1 ? `   # ${g.count} pages share ${g.template}` : "";
+    lines.push(`${path}${annotation}${note}`);
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+
+function kebab(s: string): string {
+  return s
+    .replace(/[^a-z0-9]+/gi, "-")
+    .replace(/^-+|-+$/g, "")
+    .toLowerCase();
+}
+
+/** A draft value for a form field, by input type. */
+function sampleValue(type: string): string | null {
+  switch (type) {
+    case "email":
+      return "test@example.com";
+    case "password":
+      return "Password123!";
+    case "tel":
+      return "5551234567";
+    case "number":
+      return "1";
+    case "url":
+      return "https://example.com";
+    case "search":
+    case "text":
+      return "test";
+    case "textarea":
+      return "Hello from web-tester";
+    default:
+      return null; // select / checkbox / radio / date etc. — skip
+  }
+}
+
+/** Build fill/click steps for a form. Returns null if it can't be driven. */
+function formSteps(form: PageForm): string[] | null {
+  const steps: string[] = ["wait:networkidle"];
+  let filled = 0;
+  for (const field of form.fields) {
+    if (!field.name) continue;
+    const value = sampleValue(field.tag === "textarea" ? "textarea" : field.type);
+    if (value === null) continue;
+    // Quote the attribute value so names with special characters
+    // (`user[email]`, `items[]`, dotted names) produce valid CSS.
+    const tag =
+      field.tag === "textarea"
+        ? "textarea"
+        : field.tag === "select"
+          ? "select"
+          : "input";
+    const selector = `${tag}[name="${field.name}"]`;
+    steps.push(`fill:${selector}=${value}`);
+    filled++;
+  }
+  if (filled === 0) return null;
+  steps.push("click:button[type=submit]");
+  return steps;
+}
+
+export type DraftJourney = {
+  name: string;
+  journey: {
+    description: string;
+    url: string;
+    steps: string[];
+    expectations: string[];
+    failOn: string;
+  };
+};
+
+/**
+ * Draft one journey per distinct form discovered. Forms are deduped by route
+ * template + field signature so a form repeated across many pages yields a
+ * single journey. These are starting points — values and expectations need a
+ * human pass before they mean anything.
+ */
+export function buildJourneys(
+  pages: ClassifiedPage[],
+  limit: number
+): DraftJourney[] {
+  const seen = new Set<string>();
+  const usedNames = new Set<string>();
+  const drafts: DraftJourney[] = [];
+
+  for (const page of pages) {
+    if (!page.ok) continue;
+    for (const form of page.forms) {
+      const signature = `${page.template}::${form.fields
+        .map((f) => `${f.name}:${f.tag}:${f.type}`)
+        .sort()
+        .join(",")}`;
+      if (seen.has(signature)) continue;
+      const steps = formSteps(form);
+      if (!steps) continue;
+      seen.add(signature);
+
+      const path = page.finalPath || page.path;
+      let base =
+        page.type === "auth"
+          ? /sign-?up|register/i.test(path)
+            ? "signup"
+            : "login"
+          : kebab(path) || "form";
+      base = `${base}-form`.replace(/-form-form$/, "-form");
+      let name = base;
+      let n = 2;
+      while (usedNames.has(name)) name = `${base}-${n++}`;
+      usedNames.add(name);
+
+      drafts.push({
+        name,
+        journey: {
+          description: `Auto-generated by \`web-tester map\` from the form on ${path}. Review the selectors, values, and add expectations before relying on it.`,
+          url: path,
+          steps,
+          expectations: [],
+          failOn: "http-5xx"
+        }
+      });
+      if (drafts.length >= limit) return drafts;
+    }
+  }
+  return drafts;
+}
+
+const RECIPES_START = "<!-- web-tester:map:start -->";
+const RECIPES_END = "<!-- web-tester:map:end -->";
+
+const TYPE_LABEL: Record<string, string> = {
+  home: "Homepage",
+  list: "List / index page",
+  detail: "Detail page",
+  form: "Form page",
+  auth: "Auth page",
+  search: "Search page",
+  content: "Content page"
+};
+
+/** Build the marker-fenced "Generated by map" recipe section. */
+export function buildRecipesSection(groups: RouteGroup[]): string {
+  const lines: string[] = [
+    RECIPES_START,
+    "",
+    "## Generated by `web-tester map`",
+    "",
+    "One smoke recipe per route template discovered. Re-running `web-tester map`",
+    "refreshes this section. Promote the useful ones into the project-specific",
+    "section above (they won't be overwritten there).",
+    "",
+    "### Sweep every discovered route",
+    "",
+    "```bash",
+    "web-tester sweep --preset map --fail-on http-5xx",
+    "```",
+    ""
+  ];
+
+  for (const g of groups) {
+    if (g.type === "error" || !g.representative.ok) continue;
+    const path = g.representative.finalPath || g.representative.path;
+    const pack = packForFacts(g.representative);
+    const expects: string[] = [];
+    if (pack === "homepage" || pack === "static") {
+      expects.push("--expect \"selector=header\"", "--expect \"selector=footer\"");
+    } else if (pack === "category") {
+      expects.push(
+        "--expect \"selector=main\"",
+        "--expect \"selector=main a[href^='/']:has(img)\""
+      );
+    } else if (pack === "has-main") {
+      expects.push("--expect \"selector=main\"");
+    } else if (pack === "has-h1") {
+      expects.push("--expect \"selector=h1\"");
+    }
+    const label = TYPE_LABEL[g.type] ?? "Page";
+    const cmd = [
+      `web-tester inspect "${path}" \\`,
+      "  --step wait:networkidle --quick \\",
+      ...expects.map((e) => `  ${e} \\`),
+      "  --fail-on http-5xx"
+    ].join("\n");
+    lines.push(
+      `### ${label} — \`${g.template}\``,
+      "",
+      `**When:** verifying \`${g.template}\` still renders.`,
+      "",
+      "```bash",
+      cmd,
+      "```",
+      ""
+    );
+  }
+
+  lines.push(RECIPES_END);
+  return lines.join("\n");
+}
+
+/**
+ * Merge the generated recipe section into an existing recipes.md body,
+ * replacing a previous generated block if present. Returns the full file.
+ */
+export function mergeRecipes(existing: string | null, section: string): string {
+  if (!existing) {
+    return `# Recipes\n\nCopy-paste one-liners for common web-tester runs.\n\n${section}\n`;
+  }
+  const start = existing.indexOf(RECIPES_START);
+  const end = existing.indexOf(RECIPES_END);
+  if (start !== -1 && end !== -1 && end > start) {
+    const before = existing.slice(0, start);
+    const after = existing.slice(end + RECIPES_END.length);
+    return `${before}${section}${after}`;
+  }
+  const sep = existing.endsWith("\n") ? "\n" : "\n\n";
+  return `${existing}${sep}${section}\n`;
+}