penthera 1.2.0 → 1.4.0

package/src/utils/http.js

@@ -1,26 +1,210 @@
+/**
+ * The one HTTP client every Penthera probe uses.
+ *
+ * It owns three things so no module has to:
+ *   - timeouts (one AbortController per request),
+ *   - the User-Agent (`penthera/<version> (+repo url)`, overridable),
+ *   - egress policy: which hosts a scan may talk to.
+ *
+ * A scan runs inside an egress context (see `runWithEgress`). The context
+ * names the scan target and records every host contacted, every request it
+ * refused, and every probe request that failed. Under `offline: true` the
+ * client refuses any host other than the target; a redirect may only move to
+ * a host on the target's registrable domain. Outside a context (unit tests,
+ * library use) requests are unrestricted and nothing is recorded.
+ */
+import { AsyncLocalStorage } from "node:async_hooks";
+import { readFileSync } from "node:fs";
+import { registrableDomain } from "./url.js";
+
 const DEFAULT_TIMEOUT = 8_000;
+const MAX_REDIRECTS = 10;
+const REPO_URL = "https://github.com/danoszz/penthera";
+
+const pkg = JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf-8"));
+
+export const DEFAULT_USER_AGENT = `penthera/${pkg.version} (+${REPO_URL})`;
+
+const store = new AsyncLocalStorage();
+
+/** Thrown (and recorded) when the egress policy refuses a request. */
+export class EgressBlockedError extends Error {
+  constructor(url, reason) {
+    super(`blocked by --offline: ${reason}`);
+    this.name = "EgressBlockedError";
+    this.code = "EGRESS_BLOCKED";
+    this.url = url;
+  }
+}
 
 /**
- * Fetch with timeout and graceful failure (returns null on network error).
+ * Create the per-scan egress context.
+ *
+ * @param {object} opts
+ * @param {string} opts.target - scan target URL; its host is always allowed
+ * @param {boolean} [opts.offline] - refuse every host except the target
+ * @param {string} [opts.userAgent] - override the default User-Agent
+ * @param {string[]} [opts.allowHosts] - extra hosts to allow under offline
  */
-export async function safeFetch(url, opts = {}) {
+export function createEgressContext(opts = {}) {
+  const targetHost = opts.target ? new URL(opts.target).hostname.toLowerCase() : null;
+  return {
+    targetHost,
+    targetDomain: targetHost ? registrableDomain(targetHost) : null,
+    offline: opts.offline === true,
+    userAgent: opts.userAgent || DEFAULT_USER_AGENT,
+    allowHosts: new Set((opts.allowHosts || []).map((h) => h.toLowerCase())),
+    contacted: new Map(), // host -> request count
+    blocked: new Map(),   // host -> refused request count
+    errors: new Map(),    // `${probe}\0${message}` -> { probe, url, message, count }
+  };
+}
+
+/** Run `fn` with `ctx` as the active egress context (propagates across awaits). */
+export function runWithEgress(ctx, fn) {
+  return store.run(ctx, fn);
+}
+
+/** The active egress context, or null outside a scan. */
+export function currentEgress() {
+  return store.getStore() ?? null;
+}
+
+function isThirdParty(ctx, host) {
+  if (!ctx.targetHost) return false;
+  if (host === ctx.targetHost) return false;
+  return registrableDomain(host) !== ctx.targetDomain;
+}
+
+/**
+ * Decide whether `url` may be requested under the active policy.
+ *
+ * @param {string|URL} url
+ * @param {{ redirect?: boolean }} [opts] - true when following a redirect hop
+ * @returns {{ allowed: boolean, reason?: string }}
+ */
+export function checkEgress(url, opts = {}, ctx = currentEgress()) {
+  if (!ctx || !ctx.offline) return { allowed: true };
+  let host;
+  try {
+    host = new URL(url).hostname.toLowerCase();
+  } catch {
+    return { allowed: false, reason: "unparseable URL" };
+  }
+  if (host === ctx.targetHost || ctx.allowHosts.has(host)) return { allowed: true };
+  // A redirect may move within the target's registrable domain (apex -> www).
+  if (opts.redirect && ctx.targetDomain && !isIpLike(host) && registrableDomain(host) === ctx.targetDomain) {
+    return { allowed: true };
+  }
+  return { allowed: false, reason: `${host} is not the scan target (${ctx.targetHost})` };
+}
+
+function isIpLike(host) {
+  return /^\d{1,3}(\.\d{1,3}){3}$/.test(host) || host.includes(":") || host.startsWith("[");
+}
+
+function bump(map, key) {
+  map.set(key, (map.get(key) || 0) + 1);
+}
+
+function recordError(ctx, probe, url, message) {
+  if (!ctx || probe === false) return;
+  const key = `${probe}\0${message}`;
+  const existing = ctx.errors.get(key);
+  if (existing) existing.count++;
+  else ctx.errors.set(key, { probe, url: String(url), message, count: 1 });
+}
+
+function withUserAgent(headers, userAgent) {
+  const h = new Headers(headers || {});
+  if (!h.has("user-agent")) h.set("user-agent", userAgent);
+  return h;
+}
+
+function describeError(err, timeoutMs) {
+  if (err?.name === "AbortError" || err?.name === "TimeoutError") return `timeout after ${timeoutMs}ms`;
+  const cause = err?.cause?.code || err?.cause?.message;
+  return cause ? `${err.message} (${cause})` : String(err?.message || err);
+}
+
+/**
+ * Issue one request through the egress policy.
+ *
+ * Resolves to the `Response` (3xx included when `redirect` is "manual", the
+ * default) and throws on refusal, timeout or network error. Failures are
+ * recorded on the active context under `opts.probe`.
+ *
+ * @param {string|URL} url
+ * @param {object} [opts] - fetch options plus `timeout` (ms) and `probe` (name
+ *   for result.errors; `false` to not record failures, e.g. when a fallback exists)
+ */
+export async function request(url, opts = {}) {
+  const ctx = currentEgress();
+  const { timeout = DEFAULT_TIMEOUT, probe = "http", redirect = "manual", signal, ...fetchOpts } = opts;
+  const userAgent = ctx?.userAgent || DEFAULT_USER_AGENT;
+
   const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), opts.timeout || DEFAULT_TIMEOUT);
+  const timer = setTimeout(() => controller.abort(), timeout);
+  const combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
+
+  let current = String(url);
+  let init = { ...fetchOpts, headers: withUserAgent(fetchOpts.headers, userAgent) };
+
   try {
-    return await fetch(url, {
-      ...opts,
-      signal: controller.signal,
-      redirect: opts.redirect ?? "manual",
-    });
+    for (let hop = 0; ; hop++) {
+      const verdict = checkEgress(current, { redirect: hop > 0 }, ctx);
+      if (!verdict.allowed) {
+        const blockedHost = safeHost(current);
+        if (ctx && blockedHost) bump(ctx.blocked, blockedHost);
+        throw new EgressBlockedError(current, verdict.reason);
+      }
+
+      const res = await globalThis.fetch(current, { ...init, signal: combined, redirect: "manual" });
+      if (ctx) bump(ctx.contacted, new URL(current).hostname.toLowerCase());
+
+      const location = res.headers.get("location");
+      const isRedirect = res.status >= 300 && res.status < 400 && location;
+      if (redirect !== "follow" || !isRedirect) return res;
+      if (hop >= MAX_REDIRECTS) throw new Error(`too many redirects (${MAX_REDIRECTS})`);
+
+      // Follow by hand so every hop passes the egress check.
+      await res.body?.cancel().catch(() => {});
+      current = new URL(location, current).toString();
+      const method = (init.method || "GET").toUpperCase();
+      if (res.status === 303 || ((res.status === 301 || res.status === 302) && method === "POST")) {
+        init = { ...init, method: "GET", body: undefined };
+      }
+    }
+  } catch (err) {
+    recordError(ctx, probe, current, err instanceof EgressBlockedError ? err.message : describeError(err, timeout));
+    throw err;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function safeHost(url) {
+  try {
+    return new URL(url).hostname.toLowerCase();
   } catch {
     return null;
-  } finally {
-    clearTimeout(timeout);
   }
 }
 
 /**
- * Fetch helper that always returns status, body text, and headers.
+ * Fetch with timeout and graceful failure (returns null on any error).
+ */
+export async function safeFetch(url, opts = {}) {
+  try {
+    return await request(url, opts);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Fetch helper that always returns status, body text, and headers
+ * (null on failure).
  */
 export async function probeFetch(url, opts = {}) {
   const res = await safeFetch(url, opts);
@@ -36,3 +220,60 @@ export async function probeFetch(url, opts = {}) {
     ok: res.ok,
   };
 }
+
+/**
+ * Fetch and read the body, never throwing. Timing covers headers and body.
+ * Failures come back as `{ status: 0, ok: false, error }`.
+ *
+ * @returns {Promise<{ status, body, size, headers, elapsed, ok, error? }>}
+ */
+export async function fetchText(url, opts = {}) {
+  const start = Date.now();
+  try {
+    const res = await request(url, opts);
+    const body = await res.text().catch(() => "");
+    return {
+      status: res.status,
+      body,
+      size: body.length,
+      headers: Object.fromEntries(res.headers.entries()),
+      elapsed: Date.now() - start,
+      ok: true,
+    };
+  } catch (err) {
+    return {
+      status: 0,
+      body: "",
+      size: 0,
+      headers: {},
+      elapsed: Date.now() - start,
+      ok: false,
+      error: err instanceof EgressBlockedError ? err.message : describeError(err, opts.timeout ?? DEFAULT_TIMEOUT),
+    };
+  }
+}
+
+/**
+ * Summarise what the active (or given) context saw, for the scan result.
+ */
+export function egressSummary(ctx = currentEgress()) {
+  if (!ctx) return null;
+  const sortHosts = (map) => [...map.entries()]
+    .map(([host, requests]) => ({ host, requests }))
+    .sort((a, b) => a.host.localeCompare(b.host));
+  const contacted = sortHosts(ctx.contacted);
+  return {
+    offline: ctx.offline,
+    userAgent: ctx.userAgent,
+    target: ctx.targetHost,
+    hosts: contacted,
+    thirdPartyHosts: contacted.filter((h) => isThirdParty(ctx, h.host)),
+    blocked: sortHosts(ctx.blocked),
+  };
+}
+
+/** Probe failures recorded on the context, as `result.errors[]`. */
+export function probeErrors(ctx = currentEgress()) {
+  if (!ctx) return [];
+  return [...ctx.errors.values()].map((e) => ({ ...e }));
+}

package/src/utils/url.js

@@ -44,6 +44,29 @@ export function redirectsToHost(locationHeader, requestUrl, attackerHost) {
   }
 }
 
+// Second-level labels that act as public suffixes under country TLDs
+// (example.co.uk, example.com.au). Not a full public-suffix list: a miss makes
+// two sibling hosts look unrelated, which errs toward blocking, not allowing.
+const SECOND_LEVEL_SUFFIXES = new Set(["ac", "co", "com", "edu", "gov", "ltd", "ne", "net", "or", "org", "plc"]);
+
+/**
+ * Registrable domain of a hostname, e.g. "app.staging.example.com" →
+ * "example.com", "shop.example.co.uk" → "example.co.uk". IPs and
+ * single-label hosts come back unchanged.
+ */
+export function registrableDomain(hostname) {
+  const host = String(hostname).toLowerCase().replace(/\.$/, "");
+  if (/^\d{1,3}(\.\d{1,3}){3}$/.test(host) || host.includes(":") || host.startsWith("[")) return host;
+  const parts = host.split(".");
+  if (parts.length <= 2) return host;
+  const tld = parts[parts.length - 1];
+  const second = parts[parts.length - 2];
+  if (tld.length === 2 && SECOND_LEVEL_SUFFIXES.has(second)) {
+    return parts.slice(-3).join(".");
+  }
+  return parts.slice(-2).join(".");
+}
+
 /** Convert a URL into a safe filename fragment. */
 export function urlToFilename(url) {
   return url