ssrwire 0.1.0

package/src/http-probe.ts

@@ -0,0 +1,461 @@
+import { createHash } from "node:crypto";
+import { performance } from "node:perf_hooks";
+
+import { createRedactionPlan, type RedactionPlan, redactText } from "./redact.js";
+import { createStreamInspector } from "./stream-parser.js";
+import type {
+  DocumentSignals,
+  HeaderSnapshot,
+  ProbeCompletion,
+  ProbeOptions,
+  ProbeResult,
+  ProbeTimings,
+  RedirectHop,
+} from "./types.js";
+
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const SNAPSHOT_HEADERS = Object.freeze([
+  "age",
+  "cache-control",
+  "cf-cache-status",
+  "content-encoding",
+  "content-language",
+  "content-length",
+  "content-type",
+  "date",
+  "etag",
+  "expires",
+  "last-modified",
+  "server",
+  "vary",
+  "x-cache",
+  "x-cache-hits",
+  "x-nextjs-cache",
+  "x-powered-by",
+  "x-vercel-cache",
+] as const);
+
+interface FinalizeOptions {
+  readonly requestedUrl: string;
+  readonly finalUrl: string;
+  readonly probe: ProbeOptions;
+  readonly redirects: readonly RedirectHop[];
+  readonly headers: HeaderSnapshot;
+  readonly timings: ProbeTimings;
+  readonly bytesRead: number;
+  readonly signals: DocumentSignals;
+  readonly completion: ProbeCompletion;
+  readonly status?: number;
+  readonly bodySha256?: string;
+  readonly error?: string;
+}
+
+function emptySignals(): DocumentSignals {
+  return {
+    descriptions: [],
+    canonicals: [],
+    robots: [],
+    h1s: [],
+    jsonLd: [],
+  };
+}
+
+function emptyHeaders(): HeaderSnapshot {
+  return { values: {}, setCookiePresent: false };
+}
+
+function elapsedSince(startedAt: number): number {
+  return Math.max(0, performance.now() - startedAt);
+}
+
+function snapshotHeaders(headers: Headers, redaction: RedactionPlan): HeaderSnapshot {
+  const values: Record<string, string> = {};
+  for (const name of SNAPSHOT_HEADERS) {
+    const value = headers.get(name);
+    if (value !== null) values[name] = redactText(value, redaction);
+  }
+  return {
+    values,
+    setCookiePresent: headers.has("set-cookie"),
+  };
+}
+
+function redactSignals(signals: DocumentSignals, redaction: RedactionPlan): DocumentSignals {
+  const element = <Signal extends { readonly value: string }>(signal: Signal): Signal => ({
+    ...signal,
+    value: redactText(signal.value, redaction),
+  });
+  return {
+    ...(signals.title === undefined ? {} : { title: element(signals.title) }),
+    ...(signals.titles === undefined ? {} : { titles: signals.titles.map(element) }),
+    descriptions: signals.descriptions.map(element),
+    canonicals: signals.canonicals.map(element),
+    robots: signals.robots.map(element),
+    h1s: signals.h1s.map(element),
+    ...(signals.firstMainText === undefined
+      ? {}
+      : { firstMainText: element(signals.firstMainText) }),
+    jsonLd: signals.jsonLd.map((signal) => ({
+      ...signal,
+      types: signal.types.map((type) => redactText(type, redaction)),
+      ...(signal.error === undefined ? {} : { error: redactText(signal.error, redaction) }),
+    })),
+    ...(signals.headClosed === undefined ? {} : { headClosed: signals.headClosed }),
+    ...(signals.bodyStarted === undefined ? {} : { bodyStarted: signals.bodyStarted }),
+    ...(signals.documentClosed === undefined ? {} : { documentClosed: signals.documentClosed }),
+  };
+}
+
+function finalize(options: FinalizeOptions): ProbeResult {
+  return {
+    requestedUrl: options.requestedUrl,
+    finalUrl: options.finalUrl,
+    agent: options.probe.agent,
+    ...(options.status === undefined ? {} : { status: options.status }),
+    redirects: options.redirects,
+    headers: options.headers,
+    timings: options.timings,
+    bytesRead: options.bytesRead,
+    ...(options.bodySha256 === undefined ? {} : { bodySha256: options.bodySha256 }),
+    signals: options.signals,
+    completion: options.completion,
+    ...(options.error === undefined ? {} : { error: options.error }),
+  };
+}
+
+function parseHttpUrl(value: string): URL | undefined {
+  try {
+    const url = new URL(value);
+    if (
+      (url.protocol !== "http:" && url.protocol !== "https:") ||
+      url.username.length > 0 ||
+      url.password.length > 0
+    ) {
+      return undefined;
+    }
+    return url;
+  } catch {
+    return undefined;
+  }
+}
+
+function safeUrlForReport(value: string): string {
+  try {
+    const url = new URL(value);
+    url.username = "";
+    url.password = "";
+    return url.href;
+  } catch {
+    return "[invalid URL]";
+  }
+}
+
+function isHtmlContentType(value: string | null): boolean {
+  if (value === null || value.trim().length === 0) return false;
+  const mediaType = value.split(";", 1)[0]?.trim().toLowerCase();
+  return mediaType === "text/html" || mediaType === "application/xhtml+xml";
+}
+
+function createRequestHeaders(
+  agentUserAgent: string,
+  customHeaders: Headers,
+  includeCustom: boolean,
+): Headers {
+  const headers = includeCustom ? new Headers(customHeaders) : new Headers();
+  if (!headers.has("accept")) {
+    headers.set("accept", "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8");
+  }
+  headers.set("accept-encoding", "identity");
+  headers.set("user-agent", agentUserAgent);
+  return headers;
+}
+
+function validateLimits(options: ProbeOptions): string | undefined {
+  if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) {
+    return "timeoutMs must be greater than zero.";
+  }
+  if (!Number.isSafeInteger(options.maxBytes) || options.maxBytes <= 0) {
+    return "maxBytes must be a positive safe integer.";
+  }
+  if (!Number.isSafeInteger(options.maxRedirects) || options.maxRedirects < 0) {
+    return "maxRedirects must be a non-negative safe integer.";
+  }
+  return undefined;
+}
+
+/**
+ * Read one URL exactly as an HTTP crawler would: no browser, no hydration, and redirects handled
+ * explicitly so caller-supplied headers cannot cross an origin boundary.
+ */
+export async function probeUrl(options: ProbeOptions): Promise<ProbeResult> {
+  const startedAt = performance.now();
+  const requestedUrl = safeUrlForReport(options.url);
+  const initialUrl = parseHttpUrl(options.url);
+  const limitError = validateLimits(options);
+  if (initialUrl === undefined || limitError !== undefined) {
+    return finalize({
+      requestedUrl,
+      finalUrl: requestedUrl,
+      probe: options,
+      redirects: [],
+      headers: emptyHeaders(),
+      timings: { headersMs: elapsedSince(startedAt) },
+      bytesRead: 0,
+      signals: emptySignals(),
+      completion: "invalid-response",
+      error:
+        initialUrl === undefined
+          ? "URL must use http:// or https:// and cannot contain embedded credentials."
+          : (limitError ?? "Probe options are invalid."),
+    });
+  }
+
+  let customHeaders: Headers;
+  try {
+    customHeaders = new Headers(options.headers);
+  } catch {
+    return finalize({
+      requestedUrl,
+      finalUrl: initialUrl.href,
+      probe: options,
+      redirects: [],
+      headers: emptyHeaders(),
+      timings: { headersMs: elapsedSince(startedAt) },
+      bytesRead: 0,
+      signals: emptySignals(),
+      completion: "invalid-response",
+      error: "One or more custom request headers are invalid.",
+    });
+  }
+
+  const redaction = createRedactionPlan(
+    options.redactHeaderValues === false
+      ? []
+      : [...customHeaders.values()].filter((value) => value.length > 0),
+  );
+  const redirects: RedirectHop[] = [];
+  const controller = new AbortController();
+  let timedOut = false;
+  const timeout = setTimeout(() => {
+    timedOut = true;
+    controller.abort();
+  }, options.timeoutMs);
+
+  let currentUrl = initialUrl;
+  let includeCustomHeaders = true;
+  let lastHeaders = emptyHeaders();
+  try {
+    while (true) {
+      const hopStartedAt = performance.now();
+      let response: Response;
+      try {
+        response = await fetch(currentUrl, {
+          method: "GET",
+          headers: createRequestHeaders(
+            options.agent.userAgent,
+            customHeaders,
+            includeCustomHeaders,
+          ),
+          redirect: "manual",
+          signal: controller.signal,
+        });
+      } catch {
+        return finalize({
+          requestedUrl,
+          finalUrl: redactText(currentUrl.href, redaction),
+          probe: options,
+          redirects,
+          headers: lastHeaders,
+          timings: { headersMs: elapsedSince(startedAt) },
+          bytesRead: 0,
+          signals: emptySignals(),
+          completion: timedOut ? "timeout" : "network-error",
+          error: timedOut ? "Request timed out." : "Network request failed.",
+        });
+      }
+
+      lastHeaders = snapshotHeaders(response.headers, redaction);
+      const responseDurationMs = elapsedSince(hopStartedAt);
+
+      if (REDIRECT_STATUSES.has(response.status)) {
+        const location = response.headers.get("location");
+        if (location === null || location.trim().length === 0) {
+          await response.body?.cancel().catch(() => undefined);
+          return finalize({
+            requestedUrl,
+            finalUrl: redactText(currentUrl.href, redaction),
+            probe: options,
+            redirects,
+            headers: lastHeaders,
+            timings: { headersMs: elapsedSince(startedAt) },
+            bytesRead: 0,
+            signals: emptySignals(),
+            completion: "invalid-response",
+            status: response.status,
+            error: "Redirect response did not include a Location header.",
+          });
+        }
+
+        let nextUrl: URL;
+        try {
+          nextUrl = new URL(location, currentUrl);
+        } catch {
+          await response.body?.cancel().catch(() => undefined);
+          return finalize({
+            requestedUrl,
+            finalUrl: redactText(currentUrl.href, redaction),
+            probe: options,
+            redirects,
+            headers: lastHeaders,
+            timings: { headersMs: elapsedSince(startedAt) },
+            bytesRead: 0,
+            signals: emptySignals(),
+            completion: "invalid-response",
+            status: response.status,
+            error: "Redirect response included an invalid Location header.",
+          });
+        }
+
+        if (
+          (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") ||
+          nextUrl.username.length > 0 ||
+          nextUrl.password.length > 0
+        ) {
+          await response.body?.cancel().catch(() => undefined);
+          return finalize({
+            requestedUrl,
+            finalUrl: redactText(currentUrl.href, redaction),
+            probe: options,
+            redirects,
+            headers: lastHeaders,
+            timings: { headersMs: elapsedSince(startedAt) },
+            bytesRead: 0,
+            signals: emptySignals(),
+            completion: "invalid-response",
+            status: response.status,
+            error:
+              nextUrl.username.length > 0 || nextUrl.password.length > 0
+                ? "Redirect target cannot contain embedded credentials."
+                : "Redirect target must use http:// or https://.",
+          });
+        }
+
+        redirects.push({
+          url: redactText(currentUrl.href, redaction),
+          status: response.status,
+          location: redactText(nextUrl.href, redaction),
+          durationMs: responseDurationMs,
+        });
+        await response.body?.cancel().catch(() => undefined);
+
+        if (redirects.length > options.maxRedirects) {
+          return finalize({
+            requestedUrl,
+            finalUrl: redactText(currentUrl.href, redaction),
+            probe: options,
+            redirects,
+            headers: lastHeaders,
+            timings: { headersMs: elapsedSince(startedAt) },
+            bytesRead: 0,
+            signals: emptySignals(),
+            completion: "invalid-response",
+            status: response.status,
+            error: `Redirect limit of ${options.maxRedirects} exceeded.`,
+          });
+        }
+
+        if (nextUrl.origin !== currentUrl.origin) includeCustomHeaders = false;
+        currentUrl = nextUrl;
+        continue;
+      }
+
+      const headersMs = elapsedSince(startedAt);
+      const contentType = response.headers.get("content-type");
+      if (!isHtmlContentType(contentType)) {
+        await response.body?.cancel().catch(() => undefined);
+        const mediaType = contentType?.split(";", 1)[0]?.trim();
+        return finalize({
+          requestedUrl,
+          finalUrl: redactText(currentUrl.href, redaction),
+          probe: options,
+          redirects,
+          headers: lastHeaders,
+          timings: { headersMs },
+          bytesRead: 0,
+          signals: emptySignals(),
+          completion: "invalid-response",
+          status: response.status,
+          error:
+            mediaType === undefined
+              ? "Expected an HTML response but the Content-Type header was missing."
+              : `Expected an HTML response but received Content-Type ${redactText(mediaType, redaction).slice(0, 160)}.`,
+        });
+      }
+      const inspector = createStreamInspector();
+      const hash = createHash("sha256");
+      let bytesRead = 0;
+      let firstByteMs: number | undefined;
+      let completion: ProbeCompletion = "complete";
+      let error: string | undefined;
+
+      if (response.body !== null) {
+        const reader = response.body.getReader();
+        try {
+          while (true) {
+            const read = await reader.read();
+            if (read.done) break;
+            if (read.value.byteLength === 0) continue;
+            if (firstByteMs === undefined) firstByteMs = elapsedSince(startedAt);
+
+            const available = options.maxBytes - bytesRead;
+            if (read.value.byteLength > available) {
+              if (available > 0) {
+                const prefix = read.value.subarray(0, available);
+                bytesRead += prefix.byteLength;
+                hash.update(prefix);
+                inspector.write(prefix, elapsedSince(startedAt));
+              }
+              completion = "max-bytes-exceeded";
+              error = `Response exceeded the ${options.maxBytes} byte limit.`;
+              await reader.cancel().catch(() => undefined);
+              break;
+            }
+
+            bytesRead += read.value.byteLength;
+            hash.update(read.value);
+            inspector.write(read.value, elapsedSince(startedAt));
+          }
+        } catch {
+          completion = timedOut ? "timeout" : "network-error";
+          error = timedOut ? "Request timed out." : "Response stream failed.";
+        } finally {
+          reader.releaseLock();
+        }
+      }
+
+      const completedAt = elapsedSince(startedAt);
+      const signals = redactSignals(inspector.end(completedAt), redaction);
+      const timings: ProbeTimings = {
+        headersMs,
+        ...(firstByteMs === undefined ? {} : { firstByteMs }),
+        ...(completion === "complete" ? { completeMs: completedAt } : {}),
+      };
+      return finalize({
+        requestedUrl,
+        finalUrl: redactText(currentUrl.href, redaction),
+        probe: options,
+        redirects,
+        headers: lastHeaders,
+        timings,
+        bytesRead,
+        bodySha256: hash.digest("hex"),
+        signals,
+        completion,
+        status: response.status,
+        ...(error === undefined ? {} : { error }),
+      });
+    }
+  } finally {
+    clearTimeout(timeout);
+  }
+}

package/src/index.ts

@@ -0,0 +1,34 @@
+export { BUILTIN_AGENTS, resolveAgent, resolveAgents } from "./agents.js";
+export { analyzeTarget, summarizeAudit } from "./analyze.js";
+export { runAudit } from "./audit.js";
+export { loadConfig, parseHeaderOption } from "./config.js";
+export { probeUrl } from "./http-probe.js";
+export { redactProbe } from "./redact.js";
+export { renderJson, renderReport, renderSarif, renderTerminal } from "./reporters.js";
+export { createStreamInspector } from "./stream-parser.js";
+export type {
+  AgentProfile,
+  AuditResult,
+  AuditSummary,
+  AuditTarget,
+  DocumentSignals,
+  ElementLocation,
+  ElementSignal,
+  Finding,
+  HeaderSnapshot,
+  JsonLdSignal,
+  ProbeCompletion,
+  ProbeOptions,
+  ProbeResult,
+  ProbeTimings,
+  RedirectHop,
+  ReportFormat,
+  RobotsAudience,
+  RobotsSignal,
+  Severity,
+  SsrWireConfig,
+  TargetAuditResult,
+  TargetExpectations,
+  TimingMark,
+} from "./types.js";
+export { VERSION } from "./version.js";

package/src/redact.ts

@@ -0,0 +1,173 @@
+import type { AuditResult, DocumentSignals, ElementSignal, ProbeResult } from "./types.js";
+
+const REDACTED = "[REDACTED]";
+
+function base64Url(value: string): string {
+  return Buffer.from(value).toString("base64url");
+}
+
+function addForms(candidates: Set<string>, value: string): void {
+  if (!value) return;
+  candidates.add(value);
+  candidates.add(encodeURIComponent(value));
+  candidates.add(new URLSearchParams({ value }).toString().slice("value=".length));
+  candidates.add(Buffer.from(value).toString("base64"));
+  candidates.add(base64Url(value));
+}
+
+function decodedBasicCredential(value: string): string | undefined {
+  const encoded = value.match(/^basic\s+([a-z0-9+/]+={0,2})$/i)?.[1];
+  if (!encoded || encoded.length % 4 === 1) return undefined;
+  const decoded = Buffer.from(encoded, "base64").toString("utf8");
+  return decoded.includes("\uFFFD") || decoded.length === 0 ? undefined : decoded;
+}
+
+function variants(value: string): readonly string[] {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    return [];
+  }
+
+  const candidates = new Set<string>();
+  addForms(candidates, trimmed);
+  const scheme = trimmed.match(/^(?:basic|bearer)\s+(.+)$/i)?.[1];
+  if (scheme) addForms(candidates, scheme);
+  const decodedBasic = decodedBasicCredential(trimmed);
+  if (decodedBasic) addForms(candidates, decodedBasic);
+
+  return [...candidates];
+}
+
+export interface RedactionPlan {
+  readonly exact: ReadonlySet<string>;
+  readonly substrings: readonly string[];
+}
+
+export function createRedactionPlan(secrets: readonly string[]): RedactionPlan {
+  const exact = new Set(secrets.flatMap(variants));
+  return {
+    exact,
+    substrings: [...exact]
+      .filter((candidate) => candidate.length >= 3)
+      .sort((a, b) => b.length - a.length),
+  };
+}
+
+export function redactText(value: string, plan: RedactionPlan): string {
+  if (plan.exact.has(value)) return REDACTED;
+  let redacted = value;
+  for (const pattern of plan.substrings) {
+    redacted = redacted.split(pattern).join(REDACTED);
+  }
+  return redacted;
+}
+
+function redactUnknown(value: unknown, plan: RedactionPlan): unknown {
+  if (typeof value === "string") {
+    return redactText(value, plan);
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => redactUnknown(item, plan));
+  }
+  if (value && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, item]) => [key, redactUnknown(item, plan)]),
+    );
+  }
+  return value;
+}
+
+function redactElement<Signal extends ElementSignal>(signal: Signal, plan: RedactionPlan): Signal {
+  return { ...signal, value: redactText(signal.value, plan) };
+}
+
+function redactSignals(signals: DocumentSignals, plan: RedactionPlan): DocumentSignals {
+  return {
+    ...(signals.title === undefined ? {} : { title: redactElement(signals.title, plan) }),
+    ...(signals.titles === undefined
+      ? {}
+      : { titles: signals.titles.map((signal) => redactElement(signal, plan)) }),
+    descriptions: signals.descriptions.map((signal) => redactElement(signal, plan)),
+    canonicals: signals.canonicals.map((signal) => redactElement(signal, plan)),
+    robots: signals.robots.map((signal) => redactElement(signal, plan)),
+    h1s: signals.h1s.map((signal) => redactElement(signal, plan)),
+    ...(signals.firstMainText === undefined
+      ? {}
+      : { firstMainText: redactElement(signals.firstMainText, plan) }),
+    jsonLd: signals.jsonLd.map((signal) => ({
+      ...signal,
+      types: signal.types.map((type) => redactText(type, plan)),
+      ...(signal.error === undefined ? {} : { error: redactText(signal.error, plan) }),
+    })),
+    ...(signals.headClosed === undefined ? {} : { headClosed: signals.headClosed }),
+    ...(signals.bodyStarted === undefined ? {} : { bodyStarted: signals.bodyStarted }),
+    ...(signals.documentClosed === undefined ? {} : { documentClosed: signals.documentClosed }),
+  };
+}
+
+function redactProbeWithPlan(probe: ProbeResult, plan: RedactionPlan): ProbeResult {
+  return {
+    ...probe,
+    requestedUrl: redactText(probe.requestedUrl, plan),
+    finalUrl: redactText(probe.finalUrl, plan),
+    agent: {
+      ...probe.agent,
+      label: redactText(probe.agent.label, plan),
+      userAgent: redactText(probe.agent.userAgent, plan),
+    },
+    redirects: probe.redirects.map((redirect) => ({
+      ...redirect,
+      url: redactText(redirect.url, plan),
+      location: redactText(redirect.location, plan),
+    })),
+    headers: {
+      ...probe.headers,
+      values: Object.fromEntries(
+        Object.entries(probe.headers.values).map(([name, value]) => [
+          name,
+          redactText(value, plan),
+        ]),
+      ),
+    },
+    signals: redactSignals(probe.signals, plan),
+    ...(probe.error === undefined ? {} : { error: redactText(probe.error, plan) }),
+  };
+}
+
+export function redactProbe(probe: ProbeResult, secrets: readonly string[]): ProbeResult {
+  const plan = createRedactionPlan(secrets);
+  if (plan.exact.size === 0) {
+    return probe;
+  }
+
+  return redactProbeWithPlan(probe, plan);
+}
+
+export function redactAudit(audit: AuditResult, secrets: readonly string[]): AuditResult {
+  const plan = createRedactionPlan(secrets);
+  if (plan.exact.size === 0) return audit;
+  return {
+    ...audit,
+    results: audit.results.map((result) => ({
+      target: {
+        ...result.target,
+        url: redactText(result.target.url, plan),
+        expectations: {
+          ...result.target.expectations,
+          ...(result.target.expectations.finalUrl === undefined
+            ? {}
+            : { finalUrl: redactText(result.target.expectations.finalUrl, plan) }),
+        },
+      },
+      probes: result.probes.map((probe) => redactProbeWithPlan(probe, plan)),
+      findings: result.findings.map((finding) => ({
+        ...finding,
+        message: redactText(finding.message, plan),
+        url: redactText(finding.url, plan),
+        ...(finding.evidence === undefined
+          ? {}
+          : { evidence: redactUnknown(finding.evidence, plan) as typeof finding.evidence }),
+      })),
+    })),
+  };
+}