ssrwire 0.1.0 → 0.3.0

package/src/stability.ts

@@ -0,0 +1,475 @@
+import {
+  effectiveTwitterCardSignal,
+  firstSocialSignal,
+  normalizeSocialValue,
+  OPEN_GRAPH_PROPERTIES,
+  OPEN_GRAPH_REQUIRED_PROPERTIES,
+  socialSignals,
+  TWITTER_CARD_REQUIRED_FIELDS,
+} from "./social.js";
+import type {
+  AgentProfile,
+  AgentStability,
+  AuditTarget,
+  ElementSignal,
+  Finding,
+  ProbeResult,
+  RobotsAudience,
+  RobotsSignal,
+  SocialMetadataProperty,
+  TimingStats,
+} from "./types.js";
+
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const ROBOTS_OPPOSITES = [
+  ["index", "noindex"],
+  ["follow", "nofollow"],
+  ["archive", "noarchive"],
+  ["snippet", "nosnippet"],
+  ["translate", "notranslate"],
+  ["imageindex", "noimageindex"],
+] as const;
+
+export interface StabilityAnalysis {
+  readonly stability: readonly AgentStability[];
+  readonly findings: readonly Finding[];
+}
+
+interface SocialSignature {
+  openGraph?: unknown;
+  twitterCard?: unknown;
+}
+
+function normalizeText(value: string): string {
+  return value.trim().replace(/\s+/g, " ");
+}
+
+function normalizeUrl(value: string, baseUrl?: string): string {
+  try {
+    const url = baseUrl === undefined ? new URL(value) : new URL(value, baseUrl);
+    url.hash = "";
+    return url.href;
+  } catch {
+    return normalizeText(value);
+  }
+}
+
+function normalizeCanonical(value: string, baseUrl: string): string {
+  const normalized = normalizeText(value);
+  return normalized.length === 0 ? "" : normalizeUrl(normalized, baseUrl);
+}
+
+function normalizedSignalValues(
+  signals: readonly ElementSignal[],
+  normalize: (value: string) => string = normalizeText,
+): readonly string[] {
+  return [
+    ...new Set(
+      signals.map((signal) => normalize(signal.value)).filter((value) => value.length > 0),
+    ),
+  ].sort();
+}
+
+function robotsAudienceForAgent(agent: AgentProfile): RobotsAudience {
+  const key = agent.key.trim().toLowerCase();
+  if (key === "googlebot" || key === "bingbot") return key;
+  return "robots";
+}
+
+function effectiveRobotsSignals(probe: ProbeResult): readonly RobotsSignal[] {
+  const audience = robotsAudienceForAgent(probe.agent);
+  const generic = probe.signals.robots.filter((signal) => signal.audience === "robots");
+  if (audience === "robots") return generic;
+  return [...generic, ...probe.signals.robots.filter((signal) => signal.audience === audience)];
+}
+
+function normalizedRobotsValue(signals: readonly RobotsSignal[]): string {
+  const directives = new Set(
+    signals
+      .flatMap((signal) => normalizeText(signal.value).toLowerCase().split(/[;,]/))
+      .map((directive) => directive.trim())
+      .filter((directive) => directive.length > 0),
+  );
+  if (directives.delete("none")) {
+    directives.add("noindex");
+    directives.add("nofollow");
+  }
+  if (directives.delete("all")) {
+    directives.add("index");
+    directives.add("follow");
+  }
+  for (const [permissive, restrictive] of ROBOTS_OPPOSITES) {
+    if (directives.has(restrictive)) directives.delete(permissive);
+  }
+  return [...directives].sort().join(",") || "<missing>";
+}
+
+function effectiveRobotsValue(probe: ProbeResult): string {
+  return normalizedRobotsValue(effectiveRobotsSignals(probe));
+}
+
+function signalValueLocations(
+  signals: readonly ElementSignal[],
+  normalize: (value: string) => string = normalizeText,
+): readonly (readonly [string, string])[] {
+  const entries = signals
+    .map((signal) => [normalize(signal.value), signal.location] as const)
+    .filter(([value]) => value.length > 0);
+  return [...new Map(entries.map((entry) => [JSON.stringify(entry), entry])).values()].sort(
+    ([leftValue, leftLocation], [rightValue, rightLocation]) => {
+      const valueOrder = leftValue.localeCompare(rightValue);
+      return valueOrder === 0 ? leftLocation.localeCompare(rightLocation) : valueOrder;
+    },
+  );
+}
+
+function robotsValueLocations(probe: ProbeResult): readonly (readonly [string, string])[] {
+  const entries = effectiveRobotsSignals(probe)
+    .map((signal) => [normalizedRobotsValue([signal]), signal.location] as const)
+    .filter(([value]) => value !== "<missing>");
+  return [...new Map(entries.map((entry) => [JSON.stringify(entry), entry])).values()].sort(
+    (left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)),
+  );
+}
+
+function socialValuesForSignature(
+  probe: ProbeResult,
+  property: SocialMetadataProperty,
+): readonly string[] {
+  const values = [
+    ...new Set(
+      socialSignals(probe.signals, property)
+        .map((signal) => normalizeSocialValue(property, signal.value, probe.finalUrl))
+        .filter((value) => value.length > 0),
+    ),
+  ];
+  return property === "og:image" ? values : values.sort();
+}
+
+function socialLocationsForSignature(
+  probe: ProbeResult,
+  property: SocialMetadataProperty,
+): readonly (readonly [string, string])[] {
+  if (property !== "og:image") {
+    return signalValueLocations(socialSignals(probe.signals, property), (value) =>
+      normalizeSocialValue(property, value, probe.finalUrl),
+    );
+  }
+  return socialSignals(probe.signals, property)
+    .map(
+      (signal) =>
+        [normalizeSocialValue(property, signal.value, probe.finalUrl), signal.location] as const,
+    )
+    .filter(([value]) => value.length > 0);
+}
+
+function socialValueSignature(target: AuditTarget, probe: ProbeResult): unknown {
+  const signature: SocialSignature = {};
+  if (target.expectations.requireOpenGraph === true) {
+    signature.openGraph = Object.fromEntries(
+      OPEN_GRAPH_PROPERTIES.map((property) => [
+        property,
+        socialValuesForSignature(probe, property),
+      ]),
+    );
+  }
+  if (target.expectations.requireTwitterCard === true) {
+    signature.twitterCard = Object.fromEntries(
+      TWITTER_CARD_REQUIRED_FIELDS.map((field) => {
+        const signal = effectiveTwitterCardSignal(probe.signals, field);
+        return [
+          field,
+          signal === undefined
+            ? "<missing>"
+            : normalizeSocialValue(signal.property, signal.value, probe.finalUrl) || "<missing>",
+        ];
+      }),
+    );
+  }
+  return Object.keys(signature).length === 0 ? undefined : signature;
+}
+
+function socialLocationSignature(target: AuditTarget, probe: ProbeResult): unknown {
+  const signature: SocialSignature = {};
+  if (target.expectations.requireOpenGraph === true) {
+    signature.openGraph = Object.fromEntries(
+      OPEN_GRAPH_PROPERTIES.map((property) => [
+        property,
+        socialLocationsForSignature(probe, property),
+      ]),
+    );
+  }
+  if (target.expectations.requireTwitterCard === true) {
+    signature.twitterCard = Object.fromEntries(
+      TWITTER_CARD_REQUIRED_FIELDS.map((field) => {
+        const signal = effectiveTwitterCardSignal(probe.signals, field);
+        return [
+          field,
+          signal === undefined
+            ? []
+            : [
+                [
+                  normalizeSocialValue(signal.property, signal.value, probe.finalUrl),
+                  signal.location,
+                ],
+              ],
+        ];
+      }),
+    );
+  }
+  return Object.keys(signature).length === 0 ? undefined : signature;
+}
+
+function metadataValueSignature(target: AuditTarget, probe: ProbeResult): string {
+  const titles = probe.signals.titles ?? (probe.signals.title ? [probe.signals.title] : []);
+  const social = socialValueSignature(target, probe);
+  return JSON.stringify({
+    title: normalizedSignalValues(titles),
+    description: normalizedSignalValues(probe.signals.descriptions),
+    canonical: normalizedSignalValues(probe.signals.canonicals, (value) =>
+      normalizeCanonical(value, probe.finalUrl),
+    ),
+    robots: effectiveRobotsValue(probe),
+    ...(social === undefined ? {} : { social }),
+  });
+}
+
+function metadataLocationSignature(target: AuditTarget, probe: ProbeResult): string {
+  const titles = probe.signals.titles ?? (probe.signals.title ? [probe.signals.title] : []);
+  const social = socialLocationSignature(target, probe);
+  return JSON.stringify({
+    title: signalValueLocations(titles),
+    description: signalValueLocations(probe.signals.descriptions),
+    canonical: signalValueLocations(probe.signals.canonicals, (value) =>
+      normalizeCanonical(value, probe.finalUrl),
+    ),
+    robots: robotsValueLocations(probe),
+    ...(social === undefined ? {} : { social }),
+  });
+}
+
+function redirectChainSignature(probe: ProbeResult): string {
+  return JSON.stringify(
+    probe.redirects.map((redirect) => ({
+      status: redirect.status,
+      url: normalizeUrl(redirect.url),
+      location: normalizeUrl(redirect.location, redirect.url),
+    })),
+  );
+}
+
+function uniqueCount(values: readonly string[]): number {
+  return new Set(values).size;
+}
+
+export function calculateTimingStats(values: readonly number[]): TimingStats | undefined {
+  const sorted = values
+    .filter((value) => Number.isFinite(value) && value >= 0)
+    .sort((a, b) => a - b);
+  if (sorted.length === 0) return undefined;
+
+  const middle = Math.floor(sorted.length / 2);
+  const lower = sorted[middle - 1];
+  const upper = sorted[middle];
+  const medianMs =
+    sorted.length % 2 === 0 && lower !== undefined && upper !== undefined
+      ? (lower + upper) / 2
+      : (upper ?? sorted[0] ?? 0);
+  const p95Index = Math.max(0, Math.ceil(sorted.length * 0.95) - 1);
+  const minMs = sorted[0] ?? 0;
+  const maxMs = sorted[sorted.length - 1] ?? minMs;
+
+  return {
+    samples: sorted.length,
+    minMs,
+    medianMs,
+    p95Ms: sorted[p95Index] ?? maxMs,
+    maxMs,
+    spreadMs: maxMs - minMs,
+  };
+}
+
+function firstNonEmpty(signals: readonly ElementSignal[]): ElementSignal | undefined {
+  return signals.find((signal) => normalizeText(signal.value).length > 0);
+}
+
+export function criticalSignalsArrivalMs(
+  target: AuditTarget,
+  probe: ProbeResult,
+): number | undefined {
+  const marks: number[] = [];
+  const { expectations } = target;
+  let required = 0;
+
+  const addRequired = (signal: ElementSignal | undefined): boolean => {
+    required += 1;
+    if (signal === undefined || normalizeText(signal.value).length === 0) return false;
+    marks.push(signal.atMs);
+    return true;
+  };
+
+  if (expectations.requireTitle && !addRequired(probe.signals.title)) return undefined;
+  if (expectations.requireDescription && !addRequired(firstNonEmpty(probe.signals.descriptions))) {
+    return undefined;
+  }
+  if (expectations.requireCanonical) {
+    const canonical = probe.signals.canonicals.find(
+      (signal) => normalizeCanonical(signal.value, probe.finalUrl).length > 0,
+    );
+    if (!addRequired(canonical)) return undefined;
+  }
+  if (expectations.requireH1 && !addRequired(firstNonEmpty(probe.signals.h1s))) return undefined;
+  if (expectations.requireMainText && !addRequired(probe.signals.firstMainText)) return undefined;
+  if (expectations.requireOpenGraph === true) {
+    for (const property of OPEN_GRAPH_REQUIRED_PROPERTIES) {
+      if (!addRequired(firstSocialSignal(probe.signals, property))) return undefined;
+    }
+  }
+  if (expectations.requireTwitterCard === true) {
+    for (const field of TWITTER_CARD_REQUIRED_FIELDS) {
+      if (!addRequired(effectiveTwitterCardSignal(probe.signals, field))) return undefined;
+    }
+  }
+
+  return required === 0 ? undefined : Math.max(...marks);
+}
+
+function addStats(
+  values: readonly number[],
+  key: "headers" | "firstByte" | "criticalSignals" | "complete",
+  target: Record<string, TimingStats>,
+): void {
+  const stats = calculateTimingStats(values);
+  if (stats !== undefined) target[key] = stats;
+}
+
+function summarizeAgent(target: AuditTarget, probes: readonly ProbeResult[]): AgentStability {
+  const agent = probes[0]?.agent;
+  if (agent === undefined) throw new Error("Cannot summarize an empty probe group.");
+  const complete = probes.filter((probe) => probe.completion === "complete");
+  const finalResponses = probes.filter(
+    (probe) => probe.status !== undefined && !REDIRECT_STATUSES.has(probe.status),
+  );
+  const timings: Record<string, TimingStats> = {};
+
+  addStats(
+    finalResponses.map((probe) => probe.timings.headersMs),
+    "headers",
+    timings,
+  );
+  addStats(
+    probes.flatMap((probe) =>
+      probe.timings.firstByteMs === undefined ? [] : [probe.timings.firstByteMs],
+    ),
+    "firstByte",
+    timings,
+  );
+  addStats(
+    complete.flatMap((probe) => {
+      const value = criticalSignalsArrivalMs(target, probe);
+      return value === undefined ? [] : [value];
+    }),
+    "criticalSignals",
+    timings,
+  );
+  addStats(
+    complete.flatMap((probe) =>
+      probe.timings.completeMs === undefined ? [] : [probe.timings.completeMs],
+    ),
+    "complete",
+    timings,
+  );
+
+  return {
+    agent,
+    samples: probes.length,
+    complete: complete.length,
+    incomplete: probes.length - complete.length,
+    timings,
+    variants: {
+      completion: uniqueCount(probes.map((probe) => probe.completion)),
+      status: uniqueCount(finalResponses.map((probe) => String(probe.status))),
+      finalUrl: uniqueCount(finalResponses.map((probe) => normalizeUrl(probe.finalUrl))),
+      redirectChain: uniqueCount(probes.map(redirectChainSignature)),
+      bodySha256: uniqueCount(complete.map((probe) => probe.bodySha256 ?? "<missing>")),
+      metadataValues: uniqueCount(complete.map((probe) => metadataValueSignature(target, probe))),
+      metadataLocations: uniqueCount(
+        complete.map((probe) => metadataLocationSignature(target, probe)),
+      ),
+    },
+  };
+}
+
+function finding(
+  target: AuditTarget,
+  summary: AgentStability,
+  code: "response-instability" | "stream-instability",
+  severity: "info" | "warning",
+  fields: readonly string[],
+): Finding {
+  const variants = summary.variants;
+  return {
+    code,
+    severity,
+    message:
+      code === "response-instability"
+        ? `${summary.agent.label} returned inconsistent HTTP response evidence across samples.`
+        : `${summary.agent.label} returned inconsistent streamed HTML evidence across samples.`,
+    url: target.url,
+    agent: summary.agent.key,
+    evidence: {
+      samples: summary.samples,
+      completeSamples: summary.complete,
+      fields: fields.join(", "),
+      variantCounts: fields
+        .map((field) => `${field}=${variants[field as keyof typeof variants]}`)
+        .join("; "),
+    },
+  };
+}
+
+function findingsFor(target: AuditTarget, summary: AgentStability): readonly Finding[] {
+  const findings: Finding[] = [];
+  const responseFields = (["completion", "status", "finalUrl", "redirectChain"] as const).filter(
+    (field) => summary.variants[field] > 1,
+  );
+  if (responseFields.length > 0) {
+    findings.push(finding(target, summary, "response-instability", "warning", responseFields));
+  }
+
+  const streamFields = (["bodySha256", "metadataValues", "metadataLocations"] as const).filter(
+    (field) => summary.variants[field] > 1,
+  );
+  if (streamFields.length > 0) {
+    const metadataChanged = streamFields.some((field) => field !== "bodySha256");
+    findings.push(
+      finding(
+        target,
+        summary,
+        "stream-instability",
+        metadataChanged ? "warning" : "info",
+        streamFields,
+      ),
+    );
+  }
+
+  return findings;
+}
+
+export function analyzeStability(
+  target: AuditTarget,
+  probes: readonly ProbeResult[],
+): StabilityAnalysis {
+  const groups = new Map<string, ProbeResult[]>();
+  for (const probe of probes) {
+    const group = groups.get(probe.agent.key) ?? [];
+    group.push(probe);
+    groups.set(probe.agent.key, group);
+  }
+
+  const stability = [...groups.values()].map((group) => summarizeAgent(target, group));
+  return {
+    stability,
+    findings: stability.flatMap((summary) => findingsFor(target, summary)),
+  };
+}

package/src/stream-parser.ts

@@ -1,5 +1,7 @@
 import { Parser } from "htmlparser2";
 
+import { isSocialMetadataProperty } from "./social.js";
+
 import type {
   DocumentSignals,
   ElementLocation,
@@ -7,6 +9,8 @@ import type {
   JsonLdSignal,
   RobotsAudience,
   RobotsSignal,
+  SocialMetadataProperty,
+  SocialMetadataSignal,
   TimingMark,
 } from "./types.js";
 
@@ -93,6 +97,8 @@ class HtmlStreamInspector implements StreamInspector {
   readonly #descriptions: ElementSignal[] = [];
   readonly #canonicals: ElementSignal[] = [];
   readonly #robots: RobotsSignal[] = [];
+  readonly #socialMetadata: SocialMetadataSignal[] = [];
+  readonly #socialMetadataCounts = new Map<SocialMetadataProperty, number>();
   readonly #h1s: ElementSignal[] = [];
   readonly #jsonLd: JsonLdSignal[] = [];
   readonly #titles: ElementSignal[] = [];
@@ -170,6 +176,7 @@ class HtmlStreamInspector implements StreamInspector {
       descriptions: [...this.#descriptions],
       canonicals: [...this.#canonicals],
       robots: [...this.#robots],
+      socialMetadata: [...this.#socialMetadata],
       h1s: [...this.#h1s],
       ...(firstMainText === undefined || firstMainText.value.length === 0 ? {} : { firstMainText }),
       jsonLd: [...this.#jsonLd],
@@ -213,7 +220,14 @@ class HtmlStreamInspector implements StreamInspector {
       return;
     }
 
-    const { name: metaNameAttribute, content, rel, href, type: scriptType } = attributes;
+    const {
+      name: metaNameAttribute,
+      property: metaPropertyAttribute,
+      content,
+      rel,
+      href,
+      type: scriptType,
+    } = attributes;
     if (name === "head") this.#inHead = true;
     if (name === "body") {
       if (this.#bodyStarted === undefined) this.#bodyStarted = this.#mark();
@@ -232,6 +246,7 @@ class HtmlStreamInspector implements StreamInspector {
 
     if (name === "meta") {
       const metaName = metaNameAttribute?.trim().toLowerCase();
+      const metaProperty = metaPropertyAttribute?.trim().toLowerCase();
       if (
         content !== undefined &&
         metaName === "description" &&
@@ -249,6 +264,21 @@ class HtmlStreamInspector implements StreamInspector {
           audience: metaName,
         });
       }
+      const socialProperty = isSocialMetadataProperty(metaProperty)
+        ? metaProperty
+        : isSocialMetadataProperty(metaName)
+          ? metaName
+          : undefined;
+      if (content !== undefined && socialProperty !== undefined) {
+        const count = this.#socialMetadataCounts.get(socialProperty) ?? 0;
+        if (count < SIGNAL_LIMIT) {
+          this.#socialMetadata.push({
+            ...this.#elementSignal(content, location),
+            property: socialProperty,
+          });
+          this.#socialMetadataCounts.set(socialProperty, count + 1);
+        }
+      }
     }
 
     if (name === "link") {

package/src/types.ts

@@ -18,6 +18,21 @@ export interface RobotsSignal extends ElementSignal {
   readonly audience: RobotsAudience;
 }
 
+export type SocialMetadataProperty =
+  | "og:title"
+  | "og:type"
+  | "og:url"
+  | "og:image"
+  | "og:description"
+  | "twitter:card"
+  | "twitter:title"
+  | "twitter:description"
+  | "twitter:image";
+
+export interface SocialMetadataSignal extends ElementSignal {
+  readonly property: SocialMetadataProperty;
+}
+
 export interface JsonLdSignal extends TimingMark {
   readonly location: ElementLocation;
   readonly valid?: boolean;
@@ -33,6 +48,8 @@ export interface DocumentSignals {
   readonly descriptions: readonly ElementSignal[];
   readonly canonicals: readonly ElementSignal[];
   readonly robots: readonly RobotsSignal[];
+  /** Present on probes produced by SSRWire 0.3.0 and newer. */
+  readonly socialMetadata?: readonly SocialMetadataSignal[];
   readonly h1s: readonly ElementSignal[];
   readonly firstMainText?: ElementSignal;
   readonly jsonLd: readonly JsonLdSignal[];
@@ -97,6 +114,8 @@ export interface ProbeResult {
   readonly signals: DocumentSignals;
   readonly completion: ProbeCompletion;
   readonly error?: string;
+  /** One-based audit sample number. Low-level probeUrl() calls leave this unset. */
+  readonly sample?: number;
 }
 
 export interface TargetExpectations {
@@ -107,6 +126,10 @@ export interface TargetExpectations {
   readonly requireCanonical: boolean;
   readonly requireH1: boolean;
   readonly requireMainText: boolean;
+  /** Require the four Open Graph protocol basic metadata properties. Defaults to false. */
+  readonly requireOpenGraph?: boolean;
+  /** Require SSRWire's Twitter Card readiness contract. Defaults to false. */
+  readonly requireTwitterCard?: boolean;
   readonly maxFirstByteMs?: number;
   readonly maxCriticalMs?: number;
 }
@@ -123,6 +146,43 @@ export interface SsrWireConfig {
   readonly timeoutMs: number;
   readonly maxBytes: number;
   readonly maxRedirects: number;
+  /** Total samples per target and agent. Defaults to one for programmatic callers. */
+  readonly repeat?: number;
+}
+
+export interface TimingStats {
+  readonly samples: number;
+  readonly minMs: number;
+  readonly medianMs: number;
+  readonly p95Ms: number;
+  readonly maxMs: number;
+  readonly spreadMs: number;
+}
+
+export interface StabilityTimings {
+  readonly headers?: TimingStats;
+  readonly firstByte?: TimingStats;
+  readonly criticalSignals?: TimingStats;
+  readonly complete?: TimingStats;
+}
+
+export interface StabilityVariants {
+  readonly completion: number;
+  readonly status: number;
+  readonly finalUrl: number;
+  readonly redirectChain: number;
+  readonly bodySha256: number;
+  readonly metadataValues: number;
+  readonly metadataLocations: number;
+}
+
+export interface AgentStability {
+  readonly agent: AgentProfile;
+  readonly samples: number;
+  readonly complete: number;
+  readonly incomplete: number;
+  readonly timings: StabilityTimings;
+  readonly variants: StabilityVariants;
 }
 
 export interface Finding {
@@ -138,6 +198,7 @@ export interface TargetAuditResult {
   readonly target: AuditTarget;
   readonly probes: readonly ProbeResult[];
   readonly findings: readonly Finding[];
+  readonly stability?: readonly AgentStability[];
 }
 
 export interface AuditSummary {
@@ -153,6 +214,7 @@ export interface AuditResult {
   readonly version: string;
   readonly generatedAt: string;
   readonly durationMs: number;
+  readonly repeat?: number;
   readonly results: readonly TargetAuditResult[];
   readonly summary: AuditSummary;
 }