tgo-wiki 0.1.0

package/packages/core/src/web/static-web-fetcher.ts

@@ -0,0 +1,537 @@
+import { createHash } from "node:crypto";
+import { lookup } from "node:dns/promises";
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { isIP } from "node:net";
+import { Readable } from "node:stream";
+import { WikiError } from "../errors.js";
+import type { StaticWebFetchResult, WebFetchConfig } from "./web-types.js";
+
+type ResolvedHost = {
+  address: string;
+  family: number;
+};
+
+type WebRequestOptions = {
+  signal: AbortSignal;
+  headers: Record<string, string>;
+  lookupHost: (hostname: string) => Promise<ResolvedHost>;
+};
+
+type WebRequest = (url: URL, options: WebRequestOptions) => Promise<Response>;
+
+type StaticWebFetchDependencies = {
+  request?: WebRequest;
+  resolveHost?: (hostname: string) => Promise<ResolvedHost[]>;
+};
+
+const maxRedirects = 5;
+
+export async function staticWebFetch(
+  url: string,
+  config: WebFetchConfig,
+  dependencies: StaticWebFetchDependencies = {}
+): Promise<StaticWebFetchResult> {
+  let currentUrl = parseHttpUrl(url);
+  const request = dependencies.request ?? nodeWebRequest;
+  const resolveHost = dependencies.resolveHost ?? resolveHostWithDns;
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs);
+  const hostPolicy = createHostPolicy(config);
+
+  try {
+    let response: Response;
+
+    for (let redirectCount = 0; ; redirectCount += 1) {
+      assertAllowedUrlHost(currentUrl, hostPolicy);
+      response = await request(currentUrl, {
+        signal: controller.signal,
+        headers: { accept: "text/html,application/xhtml+xml" },
+        lookupHost: hostname => resolveAllowedHost(currentUrl, hostname, hostPolicy, resolveHost)
+      });
+
+      const redirectTarget = redirectLocation(response, currentUrl);
+      if (!redirectTarget) {
+        break;
+      }
+
+      await cancelResponseBody(response);
+
+      if (redirectCount >= maxRedirects) {
+        throw new WikiError("fetch_failed", `Web fetch exceeded redirect limit: ${url}`, {
+          url,
+          maxRedirects
+        });
+      }
+
+      currentUrl = redirectTarget;
+    }
+
+    const contentType = response.headers.get("content-type") ?? "";
+
+    if (!response.ok) {
+      await cancelResponseBody(response);
+      throw new WikiError("fetch_failed", `Web fetch returned HTTP ${response.status}`, {
+        url,
+        statusCode: response.status
+      });
+    }
+
+    if (!isHtmlContentType(contentType)) {
+      await cancelResponseBody(response);
+      throw new WikiError("fetch_failed", `Web fetch did not return HTML: ${contentType || "unknown"}`, {
+        url,
+        contentType
+      });
+    }
+
+    const bytes = await readResponseBytes(response, config.maxResponseBytes);
+    const htmlText = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+
+    return {
+      requestedUrl: parseHttpUrl(url).toString(),
+      finalUrl: currentUrl.toString(),
+      contentType,
+      statusCode: response.status,
+      htmlBytes: bytes.byteLength,
+      htmlContent: bytes,
+      htmlText,
+      htmlBlobSha256: createHash("sha256").update(bytes).digest("hex")
+    };
+  } catch (error) {
+    if (error instanceof WikiError) {
+      throw error;
+    }
+
+    if (error instanceof Error && error.name === "AbortError") {
+      throw new WikiError("fetch_timeout", `Web fetch timed out: ${url}`, {
+        url,
+        requestTimeoutMs: config.requestTimeoutMs
+      });
+    }
+
+    throw new WikiError("fetch_failed", error instanceof Error ? error.message : String(error), { url });
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+async function nodeWebRequest(url: URL, options: WebRequestOptions): Promise<Response> {
+  return new Promise<Response>((resolve, reject) => {
+    let settled = false;
+    const resolveOnce = (response: Response) => {
+      settled = true;
+      resolve(response);
+    };
+    const rejectOnce = (error: Error) => {
+      if (!settled) {
+        settled = true;
+        reject(error);
+      }
+    };
+    const request = (url.protocol === "https:" ? httpsRequest : httpRequest)(
+      url,
+      {
+        method: "GET",
+        headers: options.headers,
+        lookup: (hostname, _lookupOptions, callback) => {
+          options.lookupHost(String(hostname)).then(
+            resolved => callback(null, resolved.address, resolved.family),
+            error => callback(error instanceof Error ? error : new Error(String(error)), "", 4)
+          );
+        }
+      },
+      message => {
+        const status = message.statusCode ?? 500;
+        const headers = headersFromMessage(message);
+        const body = [204, 205, 304].includes(status)
+          ? null
+          : (Readable.toWeb(message) as unknown as ReadableStream<Uint8Array>);
+        resolveOnce(new Response(body, { status, headers }));
+      }
+    );
+
+    const abort = () => {
+      const error = new Error("Web fetch aborted");
+      error.name = "AbortError";
+      request.destroy(error);
+      rejectOnce(error);
+    };
+
+    if (options.signal.aborted) {
+      abort();
+    } else {
+      options.signal.addEventListener("abort", abort, { once: true });
+    }
+
+    request.on("error", rejectOnce);
+    request.end();
+  });
+}
+
+function headersFromMessage(message: { headers: Record<string, string | string[] | undefined> }): Headers {
+  const headers = new Headers();
+
+  for (const [name, value] of Object.entries(message.headers)) {
+    if (Array.isArray(value)) {
+      for (const entry of value) {
+        headers.append(name, entry);
+      }
+      continue;
+    }
+
+    if (value !== undefined) {
+      headers.set(name, value);
+    }
+  }
+
+  return headers;
+}
+
+function parseHttpUrl(value: string): URL {
+  let parsed: URL;
+
+  try {
+    parsed = new URL(value);
+  } catch (error) {
+    throw new WikiError("invalid_url", `Invalid web URL: ${value}`, {
+      url: value,
+      reason: error instanceof Error ? error.message : String(error)
+    });
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new WikiError("invalid_url", `Unsupported web URL protocol: ${parsed.protocol}`, {
+      url: value,
+      protocol: parsed.protocol
+    });
+  }
+
+  return parsed;
+}
+
+function redirectLocation(response: Response, currentUrl: URL): URL | null {
+  if (![301, 302, 303, 307, 308].includes(response.status)) {
+    return null;
+  }
+
+  const location = response.headers.get("location");
+  if (!location) {
+    return null;
+  }
+
+  return parseHttpUrl(new URL(location, currentUrl).toString());
+}
+
+type HostPolicy = {
+  allowedPrivateHosts: Set<string>;
+};
+
+function createHostPolicy(config: WebFetchConfig): HostPolicy {
+  return {
+    allowedPrivateHosts: new Set((config.allowedPrivateHosts ?? []).map(normalizeHost))
+  };
+}
+
+function assertAllowedUrlHost(url: URL, policy: HostPolicy): void {
+  const host = normalizeHost(url.hostname);
+
+  if (policy.allowedPrivateHosts.has(host)) {
+    return;
+  }
+
+  if (host === "localhost" || host.endsWith(".localhost")) {
+    throw disallowedHostError(url, "localhost hostnames are not allowed");
+  }
+
+  const literalIp = hostToIpLiteral(host);
+  if (literalIp && isDisallowedIp(literalIp)) {
+    throw disallowedHostError(url, "private or reserved IP addresses are not allowed", literalIp);
+  }
+}
+
+async function resolveAllowedHost(
+  url: URL,
+  hostname: string,
+  policy: HostPolicy,
+  resolveHost: (hostname: string) => Promise<ResolvedHost[]>
+): Promise<ResolvedHost> {
+  const host = normalizeHost(hostname);
+  const allowPrivate = policy.allowedPrivateHosts.has(host);
+
+  if (!allowPrivate && (host === "localhost" || host.endsWith(".localhost"))) {
+    throw disallowedHostError(url, "localhost hostnames are not allowed");
+  }
+
+  const literalIp = hostToIpLiteral(host);
+  if (literalIp) {
+    if (!allowPrivate && isDisallowedIp(literalIp)) {
+      throw disallowedHostError(url, "private or reserved IP addresses are not allowed", literalIp);
+    }
+
+    return { address: literalIp, family: isIP(literalIp) };
+  }
+
+  let addresses: ResolvedHost[];
+  try {
+    addresses = await resolveHost(host);
+  } catch (error) {
+    throw new WikiError("fetch_failed", `Could not resolve web URL host: ${host}`, {
+      url: url.toString(),
+      host,
+      reason: error instanceof Error ? error.message : String(error)
+    });
+  }
+
+  if (addresses.length === 0) {
+    throw new WikiError("fetch_failed", `Could not resolve web URL host: ${host}`, {
+      url: url.toString(),
+      host
+    });
+  }
+
+  if (!allowPrivate) {
+    for (const address of addresses) {
+      if (isDisallowedIp(address.address)) {
+        throw disallowedHostError(url, "host resolves to a private or reserved IP address", address.address);
+      }
+    }
+  }
+
+  return addresses[0];
+}
+
+async function resolveHostWithDns(hostname: string): Promise<ResolvedHost[]> {
+  return lookup(hostname, { all: true, verbatim: true });
+}
+
+function normalizeHost(value: string): string {
+  let host = value.trim().toLowerCase();
+
+  if (host.startsWith("[") && host.endsWith("]")) {
+    host = host.slice(1, -1);
+  }
+
+  while (host.endsWith(".")) {
+    host = host.slice(0, -1);
+  }
+
+  return host;
+}
+
+function hostToIpLiteral(host: string): string | null {
+  return isIP(host) === 0 ? null : host;
+}
+
+function disallowedHostError(url: URL, reason: string, address?: string): WikiError {
+  return new WikiError("invalid_url", `Web URL target is not allowed: ${url.toString()}`, {
+    url: url.toString(),
+    reason,
+    address
+  });
+}
+
+function isDisallowedIp(address: string): boolean {
+  const normalized = normalizeHost(address);
+  const family = isIP(normalized);
+
+  if (family === 4) {
+    return isDisallowedIpv4(normalized);
+  }
+
+  if (family === 6) {
+    return isDisallowedIpv6(normalized);
+  }
+
+  return true;
+}
+
+function isDisallowedIpv4(address: string): boolean {
+  const value = ipv4ToNumber(address);
+  const ranges: Array<[string, number]> = [
+    ["0.0.0.0", 8],
+    ["10.0.0.0", 8],
+    ["100.64.0.0", 10],
+    ["127.0.0.0", 8],
+    ["169.254.0.0", 16],
+    ["172.16.0.0", 12],
+    ["192.0.0.0", 24],
+    ["192.0.2.0", 24],
+    ["192.168.0.0", 16],
+    ["198.18.0.0", 15],
+    ["198.51.100.0", 24],
+    ["203.0.113.0", 24],
+    ["224.0.0.0", 4],
+    ["240.0.0.0", 4],
+    ["255.255.255.255", 32]
+  ];
+
+  return ranges.some(([base, bits]) => ipv4InRange(value, ipv4ToNumber(base), bits));
+}
+
+function ipv4ToNumber(address: string): number {
+  const parts = address.split(".").map(part => Number.parseInt(part, 10));
+  return (((parts[0] << 24) >>> 0) + (parts[1] << 16) + (parts[2] << 8) + parts[3]) >>> 0;
+}
+
+function ipv4InRange(value: number, base: number, bits: number): boolean {
+  const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0;
+  return (value & mask) === (base & mask);
+}
+
+function isDisallowedIpv6(address: string): boolean {
+  const value = ipv6ToBigInt(address);
+  const mappedIpv4 = mappedIpv4Address(value);
+
+  if (mappedIpv4) {
+    return isDisallowedIpv4(mappedIpv4);
+  }
+
+  const ranges: Array<[string, number]> = [
+    ["::", 128],
+    ["::", 96],
+    ["::1", 128],
+    ["64:ff9b::", 96],
+    ["64:ff9b:1::", 48],
+    ["100::", 64],
+    ["2001::", 23],
+    ["2001:db8::", 32],
+    ["2002::", 16],
+    ["fc00::", 7],
+    ["fe80::", 10],
+    ["ff00::", 8]
+  ];
+
+  return ranges.some(([base, bits]) => ipv6InRange(value, ipv6ToBigInt(base), bits));
+}
+
+function mappedIpv4Address(value: bigint): string | null {
+  const mappedPrefix = ipv6ToBigInt("::ffff:0:0");
+  if (!ipv6InRange(value, mappedPrefix, 96)) {
+    return null;
+  }
+
+  const ipv4 = Number(value & 0xffffffffn);
+  return [
+    (ipv4 >>> 24) & 0xff,
+    (ipv4 >>> 16) & 0xff,
+    (ipv4 >>> 8) & 0xff,
+    ipv4 & 0xff
+  ].join(".");
+}
+
+function ipv6InRange(value: bigint, base: bigint, bits: number): boolean {
+  if (bits === 0) {
+    return true;
+  }
+
+  const mask = ((1n << BigInt(bits)) - 1n) << BigInt(128 - bits);
+  return (value & mask) === (base & mask);
+}
+
+function ipv6ToBigInt(address: string): bigint {
+  const normalized = address.toLowerCase();
+  const [head = "", tail = ""] = normalized.split("::");
+  const headParts = parseIpv6Parts(head);
+  const tailParts = parseIpv6Parts(tail);
+  const missing = 8 - headParts.length - tailParts.length;
+  const parts = normalized.includes("::") ? [...headParts, ...Array(Math.max(missing, 0)).fill(0), ...tailParts] : headParts;
+
+  return parts.reduce((value, part) => (value << 16n) + BigInt(part), 0n);
+}
+
+function parseIpv6Parts(value: string): number[] {
+  if (value.length === 0) {
+    return [];
+  }
+
+  const rawParts = value.split(":");
+  const last = rawParts.at(-1);
+
+  if (last && last.includes(".")) {
+    rawParts.pop();
+    const ipv4 = ipv4ToNumber(last);
+    rawParts.push(String((ipv4 >>> 16) & 0xffff), String(ipv4 & 0xffff));
+  }
+
+  return rawParts.map(part => Number.parseInt(part, 16));
+}
+
+function isHtmlContentType(value: string): boolean {
+  const mediaType = value.split(";", 1)[0]?.trim().toLowerCase();
+  return mediaType === "text/html" || mediaType === "application/xhtml+xml";
+}
+
+async function cancelResponseBody(response: Response): Promise<void> {
+  try {
+    await response.body?.cancel();
+  } catch {
+    // Best-effort cleanup only; callers should keep the original fetch error.
+  }
+}
+
+async function readResponseBytes(response: Response, maxBytes: number): Promise<Uint8Array> {
+  if (!response.body) {
+    const bytes = new Uint8Array(await response.arrayBuffer());
+    assertByteLimit(bytes.byteLength, maxBytes);
+    return bytes;
+  }
+
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let observedBytes = 0;
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+
+      if (done) {
+        break;
+      }
+
+      if (!value) {
+        continue;
+      }
+
+      observedBytes += value.byteLength;
+      if (observedBytes > maxBytes) {
+        const error = responseTooLargeError(observedBytes, maxBytes);
+
+        try {
+          await reader.cancel(error);
+        } catch {
+          // Best-effort cleanup only; keep response_too_large as the error.
+        }
+
+        throw error;
+      }
+
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const bytes = new Uint8Array(observedBytes);
+  let offset = 0;
+
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+
+  return bytes;
+}
+
+function assertByteLimit(observedBytes: number, maxBytes: number): void {
+  if (observedBytes > maxBytes) {
+    throw responseTooLargeError(observedBytes, maxBytes);
+  }
+}
+
+function responseTooLargeError(observedBytes: number, maxBytes: number): WikiError {
+  return new WikiError("response_too_large", `Web response exceeded byte limit: ${observedBytes} > ${maxBytes}`, {
+    observedBytes,
+    maxBytes
+  });
+}

package/packages/core/src/web/web-id.ts

@@ -0,0 +1,26 @@
+import { createHash } from "node:crypto";
+import { assertValidSourceId } from "../sources/source-id.js";
+
+export type WebSourceIdInput = {
+  finalUrl: string;
+  htmlBlobSha256: string;
+  fetchedAt: Date;
+};
+
+export type WebSourceIdGenerator = (input: WebSourceIdInput) => string;
+
+export function generateWebSourceId(input: WebSourceIdInput): string {
+  const now = input.fetchedAt;
+  const yyyy = String(now.getUTCFullYear()).padStart(4, "0");
+  const mm = String(now.getUTCMonth() + 1).padStart(2, "0");
+  const dd = String(now.getUTCDate()).padStart(2, "0");
+  const suffix = createHash("sha256")
+    .update(input.finalUrl)
+    .update("\0")
+    .update(input.htmlBlobSha256)
+    .digest("hex")
+    .slice(0, 16);
+  const id = `web_${yyyy}${mm}${dd}_${suffix}`;
+  assertValidSourceId(id);
+  return id;
+}