web-tester-for-claude 0.4.0

package/src/inspector/verdict.ts

@@ -0,0 +1,275 @@
+import type { Page } from "playwright";
+import type {
+  ConsoleEntry,
+  NetworkEntry,
+  PageErrorEntry
+} from "./capture";
+
+export type FailOnKind =
+  | "page-errors"
+  | "console-errors"
+  | "http-4xx"
+  | "http-5xx";
+
+const FAIL_ON_ALIASES: Record<string, FailOnKind> = {
+  "page-errors": "page-errors",
+  "console-errors": "console-errors",
+  "http-4xx": "http-4xx",
+  "http-5xx": "http-5xx",
+  "4xx": "http-4xx",
+  "5xx": "http-5xx"
+};
+
+export function parseFailOn(raw: string): FailOnKind[] {
+  const parts = raw
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+  const out: FailOnKind[] = [];
+  for (const p of parts) {
+    const mapped = FAIL_ON_ALIASES[p];
+    if (!mapped)
+      throw new Error(
+        `--fail-on unknown kind "${p}". Known: page-errors, console-errors, 4xx, 5xx`
+      );
+    if (!out.includes(mapped)) out.push(mapped);
+  }
+  return out;
+}
+
+export type Expectation =
+  | { kind: "text"; text: string }
+  | { kind: "no-text"; text: string }
+  | { kind: "selector"; selector: string }
+  | { kind: "no-selector"; selector: string }
+  | { kind: "attr"; name: string; value: string };
+
+export type ExpectationResult = {
+  expectation: Expectation;
+  ok: boolean;
+  detail?: string;
+};
+
+/**
+ * Parse `--expect <kind>=<value>` shorthand.
+ *
+ *   text=Welcome                      → page must contain text "Welcome"
+ *   no-text=Error                     → page must NOT contain text "Error"
+ *   selector=button[type=submit]      → element must be visible
+ *   no-selector=.error-banner         → element must not be visible
+ *   attr=Quantity:1000                → data-attr-name="Quantity" must have value or label "1000"
+ */
+export function parseExpectation(raw: string): Expectation {
+  const trimmed = raw.trim();
+  if (!trimmed) throw new Error("empty --expect");
+  const eq = trimmed.indexOf("=");
+  if (eq === -1)
+    throw new Error(
+      `--expect needs <kind>=<value> (got "${raw}"). Kinds: text, no-text, selector, no-selector, attr`
+    );
+  const kind = trimmed.slice(0, eq);
+  const value = trimmed.slice(eq + 1);
+  switch (kind) {
+    case "text":
+      if (!value) throw new Error("--expect text= needs text");
+      return { kind: "text", text: value };
+    case "no-text":
+      if (!value) throw new Error("--expect no-text= needs text");
+      return { kind: "no-text", text: value };
+    case "selector":
+      if (!value) throw new Error("--expect selector= needs a selector");
+      return { kind: "selector", selector: value };
+    case "no-selector":
+      if (!value) throw new Error("--expect no-selector= needs a selector");
+      return { kind: "no-selector", selector: value };
+    case "attr": {
+      const colon = value.indexOf(":");
+      if (colon === -1)
+        throw new Error(
+          "--expect attr= needs <name>:<value> (e.g. attr=Quantity:1000)"
+        );
+      return {
+        kind: "attr",
+        name: value.slice(0, colon),
+        value: value.slice(colon + 1)
+      };
+    }
+    default:
+      throw new Error(
+        `--expect unknown kind "${kind}" in "${raw}". Kinds: text, no-text, selector, no-selector, attr`
+      );
+  }
+}
+
+/**
+ * How long to wait for a positive assertion (text/selector visible, attr
+ * present) before deciding it failed. 5s leaves room for an element that
+ * renders a few seconds after a client-side route transition + hydration;
+ * tighter values flake on prod-like latency.
+ */
+const EXPECT_TIMEOUT_MS = 5_000;
+
+export async function evaluateExpectations(
+  page: Page,
+  expectations: Expectation[]
+): Promise<ExpectationResult[]> {
+  const results: ExpectationResult[] = [];
+  for (const e of expectations) {
+    results.push(await evaluateOne(page, e));
+  }
+  return results;
+}
+
+async function evaluateOne(
+  page: Page,
+  e: Expectation
+): Promise<ExpectationResult> {
+  try {
+    if (e.kind === "text") {
+      const found = await page
+        .getByText(e.text)
+        .first()
+        .waitFor({ state: "visible", timeout: EXPECT_TIMEOUT_MS })
+        .then(() => true)
+        .catch(() => false);
+      return {
+        expectation: e,
+        ok: found,
+        ...(found ? {} : { detail: `text "${e.text}" not visible on final page` })
+      };
+    }
+    if (e.kind === "no-text") {
+      // Slightly shorter timeout — we expect this NOT to appear.
+      const found = await page
+        .getByText(e.text)
+        .first()
+        .waitFor({ state: "visible", timeout: 1_500 })
+        .then(() => true)
+        .catch(() => false);
+      return {
+        expectation: e,
+        ok: !found,
+        ...(found
+          ? { detail: `text "${e.text}" was visible (should not be)` }
+          : {})
+      };
+    }
+    if (e.kind === "selector") {
+      const found = await page
+        .locator(e.selector)
+        .first()
+        .waitFor({ state: "visible", timeout: EXPECT_TIMEOUT_MS })
+        .then(() => true)
+        .catch(() => false);
+      return {
+        expectation: e,
+        ok: found,
+        ...(found ? {} : { detail: `selector "${e.selector}" not visible` })
+      };
+    }
+    if (e.kind === "no-selector") {
+      const found = await page
+        .locator(e.selector)
+        .first()
+        .waitFor({ state: "visible", timeout: 1_500 })
+        .then(() => true)
+        .catch(() => false);
+      return {
+        expectation: e,
+        ok: !found,
+        ...(found
+          ? { detail: `selector "${e.selector}" was visible (should not be)` }
+          : {})
+      };
+    }
+    // attr
+    const snapshot = await page.evaluate((name) => {
+      const el = document.querySelector(`[data-attr-name="${name}"]`);
+      if (!el) return null;
+      return {
+        value: el.getAttribute("data-attr-selected-value") ?? "",
+        label: el.getAttribute("data-attr-selected-label") ?? ""
+      };
+    }, e.name);
+    if (snapshot === null) {
+      return {
+        expectation: e,
+        ok: false,
+        detail: `no data-attr-name="${e.name}" found — page may need instrumentation`
+      };
+    }
+    const matches = snapshot.value === e.value || snapshot.label === e.value;
+    return {
+      expectation: e,
+      ok: matches,
+      ...(matches
+        ? {}
+        : {
+            detail: `attr "${e.name}" was value="${snapshot.value}" label="${snapshot.label}", expected "${e.value}"`
+          })
+    };
+  } catch (err) {
+    return {
+      expectation: e,
+      ok: false,
+      detail: err instanceof Error ? err.message : String(err)
+    };
+  }
+}
+
+export type VerdictInputs = {
+  failedSteps: number;
+  pageErrors: PageErrorEntry[];
+  consoleEntries: ConsoleEntry[];
+  networkEntries: NetworkEntry[];
+  expectations: ExpectationResult[];
+  failOn: FailOnKind[];
+};
+
+export type Verdict = {
+  ok: boolean;
+  triggers: string[];
+};
+
+/**
+ * Compute the final pass/fail verdict. `ok` is true iff:
+ *   - every step executed (failedSteps === 0)
+ *   - every expectation passed
+ *   - no signal from --fail-on tripped
+ *
+ * `triggers` is a list of one-line reasons the verdict failed, suitable for
+ * surfacing in the CLI summary and HTML report. Empty when ok.
+ */
+export function computeVerdict(inputs: VerdictInputs): Verdict {
+  const triggers: string[] = [];
+  if (inputs.failedSteps > 0)
+    triggers.push(`${inputs.failedSteps} step(s) failed`);
+
+  if (inputs.failOn.includes("page-errors") && inputs.pageErrors.length > 0)
+    triggers.push(`${inputs.pageErrors.length} uncaught page error(s)`);
+
+  if (inputs.failOn.includes("console-errors")) {
+    const n = inputs.consoleEntries.filter((e) => e.type === "error").length;
+    if (n > 0) triggers.push(`${n} console error(s)`);
+  }
+
+  if (inputs.failOn.includes("http-4xx")) {
+    const n = inputs.networkEntries.filter(
+      (e) => e.status !== null && e.status >= 400 && e.status < 500
+    ).length;
+    if (n > 0) triggers.push(`${n} 4xx response(s)`);
+  }
+
+  if (inputs.failOn.includes("http-5xx")) {
+    const n = inputs.networkEntries.filter(
+      (e) => e.status !== null && e.status >= 500
+    ).length;
+    if (n > 0) triggers.push(`${n} 5xx response(s)`);
+  }
+
+  const failedExp = inputs.expectations.filter((r) => !r.ok);
+  if (failedExp.length > 0)
+    triggers.push(`${failedExp.length} expectation(s) failed`);
+
+  return { ok: triggers.length === 0, triggers };
+}

package/src/journeys.ts

@@ -0,0 +1,78 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { userJourneysDir } from "./util/paths";
+
+/**
+ * A "journey" bundles a URL + step chain + assertions into a single named
+ * recipe — the same shape as a manual `inspect` invocation, but persisted
+ * to disk so it can be invoked by name (and by the `impact` command). New
+ * journeys go in `.web-tester/journeys/<name>.json` at your project root.
+ */
+export type Journey = {
+  /** Optional human description, for the CLI listing. */
+  description?: string;
+  /** Path or absolute URL; resolved against WEB_TESTER_BASE_URL if relative. */
+  url: string;
+  /** Step strings in the same grammar as `--step` (parsed with `parseStep`). */
+  steps: string[];
+  /** Expectation strings in `--expect` syntax (parsed with `parseExpectation`). */
+  expectations?: string[];
+  /** Fail-on signals (`page-errors,4xx,5xx,console-errors`). Default 5xx only. */
+  failOn?: string;
+  /** Persist-check window in ms; 0 = single check. */
+  persistMs?: number;
+};
+
+export function loadJourney(name: string): Journey {
+  const dir = userJourneysDir();
+  const path = resolve(dir, `${name}.json`);
+  if (!existsSync(path))
+    throw new Error(
+      `unknown journey "${name}". Looked for ${path}. Known: ${listJourneyNames().join(", ") || "(none — add .json files to .web-tester/journeys/)"}`
+    );
+  const raw = readFileSync(path, "utf-8");
+  const parsed = JSON.parse(raw) as Journey;
+  if (!parsed.url || !Array.isArray(parsed.steps))
+    throw new Error(`journey "${name}" is missing required "url" or "steps"`);
+  return parsed;
+}
+
+/**
+ * Persist a journey to `.web-tester/journeys/<name>.json` — the plain-text
+ * "recipe" a rerun replays. Stores only the flow (url + steps + assertions),
+ * never run artifacts. The name is slugified so it's safe as a filename.
+ * Returns the absolute path written.
+ */
+export function saveJourney(name: string, journey: Journey): string {
+  const slug = name
+    .replace(/\.json$/i, "")
+    .replace(/[^a-z0-9_-]+/gi, "-")
+    .replace(/^-+|-+$/g, "")
+    .toLowerCase();
+  if (!slug) throw new Error(`invalid journey name: "${name}"`);
+  const dir = userJourneysDir();
+  mkdirSync(dir, { recursive: true });
+  const path = resolve(dir, `${slug}.json`);
+  writeFileSync(path, `${JSON.stringify(journey, null, 2)}\n`);
+  return path;
+}
+
+export function listJourneyNames(): string[] {
+  const dir = userJourneysDir();
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((f) => f.endsWith(".json"))
+    .map((f) => f.slice(0, -".json".length))
+    .sort();
+}
+
+export function listJourneys(): Array<{ name: string; description: string }> {
+  return listJourneyNames().map((name) => {
+    try {
+      const j = loadJourney(name);
+      return { name, description: j.description ?? "" };
+    } catch {
+      return { name, description: "(failed to load)" };
+    }
+  });
+}

package/src/kb.ts

@@ -0,0 +1,84 @@
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { resolve } from "node:path";
+import { userKnowledgeDirs } from "./util/paths";
+
+export type KnowledgeFile = {
+  /** Slug derived from filename (no `.md`). */
+  topic: string;
+  /** First markdown H1 in the file, or filename if absent. */
+  title: string;
+  /** Absolute path on disk. */
+  path: string;
+  /** Tags / id parsed out of any YAML-ish frontmatter block. */
+  meta: Record<string, string | string[] | boolean>;
+};
+
+function parseSimpleFrontmatter(
+  raw: string
+): { meta: Record<string, string | string[] | boolean>; body: string } {
+  const match = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (!match || match[1] === undefined || match[2] === undefined) {
+    return { meta: {}, body: raw };
+  }
+  const meta: Record<string, string | string[] | boolean> = {};
+  for (const line of match[1].split("\n")) {
+    const kv = line.match(/^([a-zA-Z][a-zA-Z0-9_]*):\s*(.*)$/);
+    if (!kv) continue;
+    const key = kv[1] ?? "";
+    const value = (kv[2] ?? "").trim();
+    if (value === "") continue;
+    if (value === "true") meta[key] = true;
+    else if (value === "false") meta[key] = false;
+    else if (/^\[.*\]$/.test(value)) {
+      meta[key] = value
+        .slice(1, -1)
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean);
+    } else meta[key] = value;
+  }
+  return { meta, body: match[2] };
+}
+
+function firstH1(body: string): string | null {
+  const line = body.split("\n").find((l) => l.startsWith("# "));
+  return line ? line.replace(/^#\s+/, "").trim() : null;
+}
+
+/**
+ * Locate the directory that holds the user's KB markdown. Walks the
+ * candidate locations returned by `userKnowledgeDirs()` in order; first
+ * match wins. Returns null if none of the candidates exist.
+ */
+function resolveKnowledgeDir(): string | null {
+  for (const dir of userKnowledgeDirs()) {
+    if (existsSync(dir)) return dir;
+  }
+  return null;
+}
+
+export function listKnowledge(): KnowledgeFile[] {
+  const dir = resolveKnowledgeDir();
+  if (!dir) return [];
+  const files = readdirSync(dir)
+    .filter((f) => f.endsWith(".md") && f !== "README.md")
+    .sort();
+  return files.map((f) => {
+    const path = resolve(dir, f);
+    const raw = readFileSync(path, "utf-8");
+    const { meta, body } = parseSimpleFrontmatter(raw);
+    const titleMeta = typeof meta.title === "string" ? meta.title : null;
+    const title = titleMeta ?? firstH1(body) ?? f;
+    return { topic: f.replace(/\.md$/, ""), title, path, meta };
+  });
+}
+
+export function readKnowledge(topic: string): KnowledgeFile & { contents: string } {
+  const all = listKnowledge();
+  const found = all.find((k) => k.topic === topic);
+  if (!found) {
+    const known = all.map((k) => k.topic).join(", ") || "(no .md files found in .web-tester/)";
+    throw new Error(`knowledge topic "${topic}" not found. Known: ${known}`);
+  }
+  return { ...found, contents: readFileSync(found.path, "utf-8") };
+}

package/src/map/classify.ts

@@ -0,0 +1,149 @@
+import type { PageFacts } from "./crawl";
+
+export type PageType =
+  | "home"
+  | "auth"
+  | "search"
+  | "form"
+  | "list"
+  | "detail"
+  | "content"
+  | "error";
+
+export type ClassifiedPage = PageFacts & {
+  type: PageType;
+  template: string;
+};
+
+export type RouteGroup = {
+  template: string;
+  type: PageType;
+  count: number;
+  /** Best representative path (prefers a healthy page). */
+  representative: ClassifiedPage;
+  members: ClassifiedPage[];
+};
+
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const HEX_RE = /^[0-9a-f]{12,}$/i;
+
+/** Does a path segment look like a dynamic id/slug rather than a fixed route? */
+function isDynamicSegment(seg: string): boolean {
+  if (!seg) return false;
+  if (/^\d+$/.test(seg)) return true; // numeric id
+  if (UUID_RE.test(seg)) return true;
+  if (HEX_RE.test(seg)) return true;
+  // Slug ending in a numeric id, e.g. "blue-widget-12345".
+  if (/-\d{3,}$/.test(seg)) return true;
+  // A long run of digits strongly implies an id (e.g. "sku12345",
+  // "order2024001"). Deliberately conservative: version-y segments like
+  // "v2beta" or "apiv2docs" have no 4+ digit run, so they stay fixed routes —
+  // over-collapsing distinct routes into one template is worse than the
+  // reverse for a site map.
+  if (/\d{4,}/.test(seg)) return true;
+  return false;
+}
+
+/**
+ * Collapse a concrete path into a route template by replacing dynamic
+ * segments with `:id`. Query strings are dropped. `/products/12345?ref=x`
+ * and `/products/67890` both become `/products/:id`.
+ */
+export function routeTemplate(path: string): string {
+  const [pathname = "/"] = path.split("?");
+  const segments = pathname.split("/").filter(Boolean);
+  if (segments.length === 0) return "/";
+  const mapped = segments.map((s) => (isDynamicSegment(s) ? ":id" : s));
+  return `/${mapped.join("/")}`;
+}
+
+const AUTH_RE =
+  /(^|\/)(login|log-in|signin|sign-in|signup|sign-up|register|auth|forgot|password|reset)(\/|$)/i;
+
+export function classify(facts: PageFacts): PageType {
+  if (!facts.ok) return "error";
+
+  const path = facts.finalPath || facts.path;
+  const template = routeTemplate(path);
+
+  if (path === "/" || template === "/") return "home";
+
+  if (facts.passwordFields > 0 || AUTH_RE.test(path)) return "auth";
+
+  if (
+    facts.searchInputs > 0 ||
+    /(^|\/)search(\/|$)/i.test(path) ||
+    /[?&](q|query|s|search)=/.test(path)
+  )
+    return "search";
+
+  // A list/index page: several catalog-style card links, or many internal
+  // links concentrated in <main>.
+  if (facts.cardLinkCount >= 3) return "list";
+
+  // A meaningful form (more than a newsletter single-field) that isn't auth.
+  const richForm = facts.forms.some(
+    (f) => f.fields.filter((x) => x.type !== "hidden").length >= 2
+  );
+  if (richForm) return "form";
+
+  // Dynamic route → a detail/show page.
+  if (template !== path && template.includes(":id")) return "detail";
+
+  return "content";
+}
+
+export function classifyAll(pages: PageFacts[]): ClassifiedPage[] {
+  return pages.map((p) => ({
+    ...p,
+    type: classify(p),
+    template: routeTemplate(p.finalPath || p.path)
+  }));
+}
+
+/** Group classified pages by route template, picking a healthy representative. */
+export function groupByTemplate(pages: ClassifiedPage[]): RouteGroup[] {
+  const groups = new Map<string, ClassifiedPage[]>();
+  for (const p of pages) {
+    const list = groups.get(p.template);
+    if (list) list.push(p);
+    else groups.set(p.template, [p]);
+  }
+
+  const result: RouteGroup[] = [];
+  for (const [template, members] of groups) {
+    // Prefer an ok page with the shortest path as the representative.
+    const sorted = [...members].sort((a, b) => {
+      if (a.ok !== b.ok) return a.ok ? -1 : 1;
+      return a.path.length - b.path.length;
+    });
+    const representative = sorted[0]!;
+    // The group's type is its representative's type, unless that's an error
+    // but a healthy member exists.
+    result.push({
+      template,
+      type: representative.type,
+      count: members.length,
+      representative,
+      members
+    });
+  }
+
+  // Stable, useful ordering: home first, then by type, then alphabetically.
+  const typeRank: Record<PageType, number> = {
+    home: 0,
+    list: 1,
+    detail: 2,
+    form: 3,
+    auth: 4,
+    search: 5,
+    content: 6,
+    error: 7
+  };
+  result.sort((a, b) => {
+    if (typeRank[a.type] !== typeRank[b.type])
+      return typeRank[a.type] - typeRank[b.type];
+    return a.template.localeCompare(b.template);
+  });
+  return result;
+}