web-tester-for-claude 0.4.0

package/src/inspector/capture.ts

@@ -0,0 +1,293 @@
+import type { BrowserContext, Page, Request, Response } from "playwright";
+
+export type ConsoleEntry = {
+  type: string;
+  text: string;
+  location?: string;
+  timestamp: number;
+};
+
+export type NetworkEntry = {
+  method: string;
+  url: string;
+  resourceType: string;
+  status: number | null;
+  statusText: string | null;
+  durationMs: number | null;
+  fromCache: boolean;
+  failureText: string | null;
+  timestamp: number;
+  /** Request payload (truncated). Only captured under `--deep`. */
+  requestBody?: string | null;
+  /** Response payload, textual content only (truncated). `--deep` only. */
+  responseBody?: string | null;
+};
+
+export type PageErrorEntry = {
+  message: string;
+  stack: string | null;
+  timestamp: number;
+};
+
+export type CaptureBuffers = {
+  consoleEntries: ConsoleEntry[];
+  networkEntries: NetworkEntry[];
+  pageErrors: PageErrorEntry[];
+  /** Marks where each step starts so we can slice the buffers per-step. */
+  cursor: { console: number; network: number; pageErrors: number };
+  /**
+   * In-flight `--deep` response-body reads. Each resolves once it has mutated
+   * its already-pushed network entry. `flushBodies` awaits these before the
+   * context closes (a closed context can't return a body).
+   */
+  pendingBodies: Promise<void>[];
+};
+
+const TRACKED_RESOURCE_TYPES = new Set([
+  "xhr",
+  "fetch",
+  "document",
+  "websocket"
+]);
+
+const NOISY_URL_FRAGMENTS = [
+  "/__nextjs",
+  "_next/static/chunks/",
+  "_next/static/css/",
+  "_next/static/media/",
+  "/_next/data/",
+  "/_next/image",
+  "favicon.ico",
+  "googletagmanager",
+  "google-analytics",
+  "googleadservices",
+  "doubleclick",
+  "hotjar",
+  "fullstory",
+  "intercom",
+  "segment.io",
+  "cookiebot",
+  "consentcdn",
+  "hs-scripts.com",
+  "hsforms.com",
+  "hs-analytics.net",
+  "hubapi.com",
+  "hubspot.com",
+  "px.ads.linkedin",
+  "linkedin.com/li/track"
+];
+
+function isNoisyUrl(url: string): boolean {
+  return NOISY_URL_FRAGMENTS.some((frag) => url.includes(frag));
+}
+
+/**
+ * Console messages we drop by default. These are CSP `report-only` violations
+ * from third-party tags, Microsoft Clarity beacons, and similar tracker chatter
+ * — interesting to no one diagnosing an app bug. Matched against text
+ * and location URL. Pass `--all-console` (or `captureAllConsole: true`) to keep
+ * everything.
+ */
+const NOISY_CONSOLE_FRAGMENTS = [
+  "Content Security Policy",
+  "Refused to load",
+  "Refused to execute",
+  "Refused to apply",
+  "Refused to connect",
+  "googleads",
+  "google.com/pagead",
+  "googletagmanager",
+  "doubleclick",
+  "google-analytics",
+  "clarity.ms",
+  "px.ads.linkedin",
+  "facebook.com/tr",
+  "hubspot",
+  "cookiebot",
+  "consentcdn",
+  "Tracking Prevention blocked",
+  "Loading failed for the <script>",
+  "Failed to load resource"
+];
+
+function isNoisyConsole(text: string, location?: string): boolean {
+  const haystack = `${text}\n${location ?? ""}`;
+  return NOISY_CONSOLE_FRAGMENTS.some((frag) => haystack.includes(frag));
+}
+
+export function newBuffers(): CaptureBuffers {
+  return {
+    consoleEntries: [],
+    networkEntries: [],
+    pageErrors: [],
+    cursor: { console: 0, network: 0, pageErrors: 0 },
+    pendingBodies: []
+  };
+}
+
+/** Await any in-flight `--deep` body reads. Call before closing the context. */
+export async function flushBodies(buffers: CaptureBuffers): Promise<void> {
+  if (buffers.pendingBodies.length === 0) return;
+  await Promise.all(buffers.pendingBodies);
+}
+
+export function snapshotCursor(buffers: CaptureBuffers): CaptureBuffers["cursor"] {
+  return {
+    console: buffers.consoleEntries.length,
+    network: buffers.networkEntries.length,
+    pageErrors: buffers.pageErrors.length
+  };
+}
+
+export function sliceSince(
+  buffers: CaptureBuffers,
+  from: CaptureBuffers["cursor"]
+): {
+  console: ConsoleEntry[];
+  network: NetworkEntry[];
+  pageErrors: PageErrorEntry[];
+} {
+  return {
+    console: buffers.consoleEntries.slice(from.console),
+    network: buffers.networkEntries.slice(from.network),
+    pageErrors: buffers.pageErrors.slice(from.pageErrors)
+  };
+}
+
+export type AttachCaptureOptions = {
+  /** Keep every network request, not just XHR/fetch/document, and skip noise filter. */
+  allNetwork: boolean;
+  /** Keep every console line, including CSP / tracker chatter. */
+  allConsole: boolean;
+  /** Record request + (textual) response bodies, truncated. Off by default. */
+  captureBodies?: boolean;
+};
+
+/** Cap on captured body size — enough to read a JSON error, not a whole HTML doc. */
+const MAX_BODY_CHARS = 8_192;
+
+/** Content types we'll read a response body for. Skips images, fonts, binaries. */
+const TEXTUAL_CONTENT_TYPE =
+  /(application\/(json|.*\+json|xml|.*\+xml|javascript|x-www-form-urlencoded)|text\/)/i;
+
+function truncateBody(raw: string): string {
+  if (raw.length <= MAX_BODY_CHARS) return raw;
+  return `${raw.slice(0, MAX_BODY_CHARS)}… (${raw.length - MAX_BODY_CHARS} more chars)`;
+}
+
+/**
+ * Read a response body when it's textual (JSON/text/xml/js). Returns null for
+ * binary responses, empty bodies, or anything Playwright can't hand back (e.g.
+ * redirects, served-from-cache). Never throws — body capture is best-effort.
+ */
+async function readTextualBody(response: Response | null): Promise<string | null> {
+  if (!response) return null;
+  const contentType = response.headers()["content-type"] ?? "";
+  if (!TEXTUAL_CONTENT_TYPE.test(contentType)) return null;
+  try {
+    const buf = await response.body();
+    if (!buf || buf.length === 0) return null;
+    return truncateBody(buf.toString("utf-8"));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Attach console + network + pageerror listeners to a context. The returned
+ * buffers grow for the lifetime of the context.
+ *
+ * By default both streams are filtered: network keeps XHR/fetch/document and
+ * drops static asset chunks + known analytics noise; console drops CSP report
+ * violations and third-party tracker chatter (Clarity, ads, etc.). The two
+ * `all*` flags opt out independently.
+ */
+export function attachCapture(
+  context: BrowserContext,
+  page: Page,
+  opts: AttachCaptureOptions
+): CaptureBuffers {
+  const buffers = newBuffers();
+  const requestStart = new Map<Request, number>();
+
+  page.on("console", (msg) => {
+    const location = msg.location();
+    const locationStr =
+      location?.url && location.url.length > 0
+        ? `${location.url}:${location.lineNumber}:${location.columnNumber}`
+        : undefined;
+    const text = msg.text();
+    if (!opts.allConsole && isNoisyConsole(text, locationStr)) return;
+    buffers.consoleEntries.push({
+      type: msg.type(),
+      text,
+      location: locationStr,
+      timestamp: Date.now()
+    });
+  });
+
+  page.on("pageerror", (err) => {
+    buffers.pageErrors.push({
+      message: err.message,
+      stack: err.stack ?? null,
+      timestamp: Date.now()
+    });
+  });
+
+  context.on("request", (req) => {
+    requestStart.set(req, Date.now());
+  });
+
+  const finalize = (
+    req: Request,
+    response: Response | null,
+    failureText: string | null
+  ): void => {
+    if (!opts.allNetwork) {
+      if (!TRACKED_RESOURCE_TYPES.has(req.resourceType())) return;
+      if (isNoisyUrl(req.url())) return;
+    }
+    const startedAt = requestStart.get(req) ?? Date.now();
+    requestStart.delete(req);
+
+    const postData = opts.captureBodies ? req.postData() : null;
+    const entry: NetworkEntry = {
+      method: req.method(),
+      url: req.url(),
+      resourceType: req.resourceType(),
+      status: response?.status() ?? null,
+      statusText: response?.statusText() ?? null,
+      durationMs: Date.now() - startedAt,
+      fromCache: response ? response.fromServiceWorker() : false,
+      failureText,
+      timestamp: startedAt,
+      ...(opts.captureBodies
+        ? { requestBody: postData ? truncateBody(postData) : null }
+        : {})
+    };
+    // Push synchronously so per-step `sliceSince` sees the entry immediately;
+    // the response body (async) is mutated onto the same object once read, and
+    // shows up in the report — which is written after `flushBodies`.
+    buffers.networkEntries.push(entry);
+    if (opts.captureBodies) {
+      buffers.pendingBodies.push(
+        readTextualBody(response).then((body) => {
+          entry.responseBody = body;
+        })
+      );
+    }
+  };
+
+  context.on("requestfinished", (req) => {
+    void (async () => {
+      const response = await req.response().catch(() => null);
+      finalize(req, response ?? null, null);
+    })();
+  });
+
+  context.on("requestfailed", (req) => {
+    finalize(req, null, req.failure()?.errorText ?? "request failed");
+  });
+
+  return buffers;
+}

package/src/inspector/deep.ts

@@ -0,0 +1,147 @@
+import type { CDPSession, Page } from "playwright";
+
+/** One scope frame from a paused exception, variable name → rendered value. */
+export type ScopeDump = {
+  type: string;
+  vars: Record<string, string>;
+};
+
+/** An uncaught exception captured with its local scope, via the debugger. */
+export type DeepError = {
+  /** First line of the exception (e.g. `TypeError: x is not a function`). */
+  reason: string;
+  functionName: string;
+  /** Script URL + line where it threw, when known. */
+  location: string | null;
+  /** Local + closure scope at the throw site. Empty when nothing readable. */
+  scopes: ScopeDump[];
+  timestamp: number;
+};
+
+export type DeepBuffers = {
+  /** Uncaught exceptions, enriched with scope. */
+  errors: DeepError[];
+  /** Unhandled promise rejections (message text). Often missed by `pageerror`. */
+  rejections: string[];
+};
+
+/** Stop pausing once we've collected this many — a throw-loop shouldn't stall the run. */
+const MAX_DEEP_ERRORS = 25;
+
+type RemoteObject = {
+  value?: unknown;
+  description?: string;
+  type?: string;
+  preview?: { properties?: Array<{ name: string; value?: string }> };
+};
+
+/** Render a CDP RemoteObject compactly: primitive value, object preview, or type. */
+function renderRemote(obj: RemoteObject | undefined): string {
+  if (!obj) return "undefined";
+  if (obj.value !== undefined) return JSON.stringify(obj.value);
+  if (obj.preview?.properties?.length) {
+    const parts = obj.preview.properties
+      .slice(0, 8)
+      .map((p) => `${p.name}: ${p.value ?? "…"}`);
+    return `{ ${parts.join(", ")} }`;
+  }
+  return obj.description ?? obj.type ?? "?";
+}
+
+/**
+ * Attach a CDP debugger to `page` that pauses on every uncaught exception,
+ * snapshots the throwing frame's local + closure scope, then resumes
+ * immediately — so the page never deadlocks. Also records unhandled promise
+ * rejections, which Playwright's `pageerror` event doesn't surface.
+ *
+ * This is opt-in (`--deep`): pausing on exceptions adds protocol overhead and
+ * is wasted on a healthy page. Returns the growing buffers plus a `detach`.
+ */
+export async function attachDeepCapture(
+  page: Page
+): Promise<{ buffers: DeepBuffers; detach: () => Promise<void> }> {
+  const buffers: DeepBuffers = { errors: [], rejections: [] };
+  const cdp: CDPSession = await page.context().newCDPSession(page);
+
+  await cdp.send("Debugger.enable");
+  await cdp.send("Runtime.enable");
+  await cdp.send("Debugger.setPauseOnExceptions", { state: "uncaught" });
+
+  cdp.on("Debugger.paused", async (evt) => {
+    // The page's JS thread is frozen here. Whatever happens, resume in the
+    // finally so a read error can't strand the page mid-exception.
+    try {
+      if (buffers.errors.length >= MAX_DEEP_ERRORS) return;
+      const frames = (evt as { callFrames?: unknown[] }).callFrames ?? [];
+      const top = frames[0] as
+        | {
+            functionName?: string;
+            url?: string;
+            location?: { lineNumber?: number };
+            scopeChain?: Array<{
+              type?: string;
+              object?: { objectId?: string };
+            }>;
+          }
+        | undefined;
+      const data = (evt as { data?: { description?: string } }).data;
+      const reason = (data?.description ?? "(uncaught exception)").split("\n")[0]!;
+
+      const scopes: ScopeDump[] = [];
+      for (const scope of top?.scopeChain ?? []) {
+        if (scope.type !== "local" && scope.type !== "closure") continue;
+        if (!scope.object?.objectId) continue;
+        const props = await cdp
+          .send("Runtime.getProperties", {
+            objectId: scope.object.objectId,
+            ownProperties: true,
+            generatePreview: true
+          })
+          .catch(() => null);
+        if (!props) continue;
+        const vars: Record<string, string> = {};
+        for (const p of (props as { result?: Array<{ name: string; value?: RemoteObject }> }).result ?? []) {
+          if (p.name === "this" || !p.value) continue;
+          vars[p.name] = renderRemote(p.value);
+        }
+        if (Object.keys(vars).length > 0) scopes.push({ type: scope.type, vars });
+      }
+
+      // Only record exceptions we could enrich with scope — an exception with
+      // no readable variables adds nothing over the message+stack already in
+      // `pageErrors`. This keeps `deepErrors` purely the value-add: the throw
+      // site's variable state. (Rejections, which rarely have useful scope at
+      // the pause point, are covered by `rejections` below.)
+      if (scopes.length === 0) return;
+      const line = top?.location?.lineNumber;
+      buffers.errors.push({
+        reason,
+        functionName: top?.functionName || "(anonymous)",
+        location: top?.url
+          ? `${top.url}${line !== undefined ? `:${line + 1}` : ""}`
+          : null,
+        scopes,
+        timestamp: Date.now()
+      });
+    } finally {
+      await cdp.send("Debugger.resume").catch(() => {});
+    }
+  });
+
+  cdp.on("Runtime.exceptionThrown", (evt) => {
+    const details = (evt as { exceptionDetails?: { text?: string; exception?: { description?: string } } })
+      .exceptionDetails;
+    const text = details?.exception?.description ?? details?.text;
+    // Only rejections land here that the pause path doesn't already cover well.
+    if (text && /uncaught \(in promise\)/i.test(details?.text ?? "")) {
+      buffers.rejections.push(text.split("\n")[0]!);
+    }
+  });
+
+  const detach = async (): Promise<void> => {
+    await cdp.send("Debugger.disable").catch(() => {});
+    await cdp.detach().catch(() => {});
+  };
+
+  return { buffers, detach };
+}

package/src/inspector/packs.ts

@@ -0,0 +1,98 @@
+import type { Expectation } from "./verdict";
+
+/**
+ * Named "expectation packs" — bundles of `--expect` assertions tailored to
+ * a page type. A sweep URL can opt into one or more packs via inline
+ * `#pack=<name>` annotations in a URL file, and the CLI can also apply a
+ * default pack to all URLs via `--pack <name>`. Per-URL expectations are
+ * the union of global `--expect`s, global `--pack`s, and the URL's own
+ * inline packs.
+ *
+ * Built-ins below are deliberately generic — invariants that hold for almost
+ * any page of that shape. For project-specific assertions, add `--expect`
+ * flags per run or per URL, or register a new generally-useful pack here.
+ */
+export const BUILT_IN_PACKS: Record<string, Expectation[]> = {
+  /**
+   * Homepage / landing page — page chrome should render. We deliberately
+   * don't assert a heading: many sites lead with a hero or marketing block
+   * that uses an <h2> or no heading at all, and the universal signal is
+   * header + footer being present.
+   */
+  homepage: [
+    { kind: "selector", selector: "header" },
+    { kind: "selector", selector: "footer" }
+  ],
+  /**
+   * Generic content / informational page. Same shape as homepage — header
+   * + footer is the universal "the page chrome rendered" signal.
+   */
+  static: [
+    { kind: "selector", selector: "header" },
+    { kind: "selector", selector: "footer" }
+  ],
+  /**
+   * Category / index page — header + footer + at least one product/article
+   * card link. The card selector matches any `<a>` inside `<main>` that
+   * points to an internal route and contains an `<img>` — a near-universal
+   * shape for catalog cards.
+   */
+  category: [
+    { kind: "selector", selector: "header" },
+    { kind: "selector", selector: "footer" },
+    { kind: "selector", selector: "main a[href^='/']:has(img)" }
+  ],
+  /**
+   * Any "main content area present" assertion — useful as a baseline when
+   * you don't want to overspecify the page shape.
+   */
+  "has-main": [{ kind: "selector", selector: "main" }],
+  /**
+   * Any page with an `<h1>`. Common safety net for content pages where the
+   * heading is the primary "we rendered the right thing" signal.
+   */
+  "has-h1": [{ kind: "selector", selector: "h1" }]
+};
+
+export function getBuiltInPack(name: string): Expectation[] {
+  const pack = BUILT_IN_PACKS[name];
+  if (!pack) {
+    const known = Object.keys(BUILT_IN_PACKS).join(", ");
+    throw new Error(
+      `unknown pack "${name}". Built-in packs: ${known}. Add new packs in src/inspector/packs.ts.`
+    );
+  }
+  return pack;
+}
+
+export function listBuiltInPackNames(): string[] {
+  return Object.keys(BUILT_IN_PACKS);
+}
+
+/**
+ * Parse a URL-file line of the form `<path> [#pack=<name>] [#pack=<name>]…`.
+ * Returns `{ path, packs }`. Lines without an annotation get `packs: []`.
+ * Inline pack syntax is tab- or space-separated from the URL.
+ */
+export function parseUrlLine(line: string): {
+  path: string;
+  packs: string[];
+} {
+  const trimmed = line.trim();
+  // Split on whitespace into tokens. First token is the path. Each
+  // subsequent token must match `#pack=<name>`; anything else is an error
+  // (so typos don't get silently ignored).
+  const parts = trimmed.split(/\s+/);
+  const path = parts[0] ?? "";
+  const packs: string[] = [];
+  for (let i = 1; i < parts.length; i++) {
+    const part = parts[i] ?? "";
+    const match = part.match(/^#pack=(.+)$/);
+    if (!match || !match[1])
+      throw new Error(
+        `invalid URL-file annotation "${part}" on line "${line}". Expected "#pack=<name>".`
+      );
+    packs.push(match[1]);
+  }
+  return { path, packs };
+}