pi-read-page 0.1.0

package/src/cache/cache.ts

@@ -0,0 +1,265 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import path from "node:path";
+import {
+  DEFAULT_MAX_BYTES,
+  truncateHead,
+} from "@earendil-works/pi-coding-agent";
+import type { NormalizedUrl, UrlNormalization } from "../security/url-policy";
+import type { ConfidenceReport, PageMetadata } from "../types";
+
+export const CACHE_DIR = path.join(
+  homedir(),
+  ".pi",
+  "agent",
+  "caches",
+  "read-page",
+);
+export const DEFAULT_TTL_DAYS = 30;
+export const USER_ACTION_TTL_DAYS = 1;
+
+export type ReadPageCacheSource = "browser";
+export type ReadPageCacheStatus =
+  | "hit"
+  | "miss"
+  | "refresh"
+  | "refresh-failed-fresh"
+  | "stale-fallback";
+
+export type CacheMeta = {
+  version: number;
+  input_url: string;
+  url: string;
+  final_url: string;
+  cache_key: string;
+  url_sha256: string;
+  normalization: UrlNormalization;
+  source: ReadPageCacheSource;
+  extractor: "defuddle";
+  extraction: string;
+  parse_mode: string;
+  browser_profile?: "persistent" | "temporary";
+  user_action: boolean;
+  confidence: ConfidenceReport;
+  metadata: PageMetadata;
+  fetched_at: string;
+  expires_at: string;
+  ttl_days: number;
+  content_sha256: string;
+  chars: number;
+  lines: number;
+};
+
+export type CachedDocument = {
+  markdown: string;
+  meta: CacheMeta;
+  fresh: boolean;
+};
+
+export type Pagination = {
+  selected: string;
+  totalLines: number;
+  shownStart: number;
+  shownEnd: number;
+  nextOffset?: number;
+  truncated: boolean;
+  shownBytes: number;
+  totalBytes: number;
+};
+
+export function sha256(input: string): string {
+  return createHash("sha256").update(input).digest("hex");
+}
+
+function slugify(input: string, fallback: string): string {
+  const slug = input
+    .toLowerCase()
+    .replace(/[^a-z0-9._-]+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 80)
+    .replace(/-$/g, "");
+  return slug || fallback;
+}
+
+function cacheKey(url: string): string {
+  const parsed = new URL(url);
+  const urlHash = sha256(url);
+  const host = slugify(parsed.hostname, "unknown-host");
+  const pathAndSearch = `${parsed.pathname}${parsed.search}`;
+  const slug = slugify(pathAndSearch === "/" ? "root" : pathAndSearch, "root");
+  return `${host}--${slug}--${urlHash.slice(0, 12)}`;
+}
+
+export function cachePaths(url: string): {
+  dirPath: string;
+  mdPath: string;
+  metaPath: string;
+  key: string;
+  urlSha256: string;
+} {
+  const key = cacheKey(url);
+  const dirPath = path.join(CACHE_DIR, key);
+  return {
+    dirPath,
+    mdPath: path.join(dirPath, "content.md"),
+    metaPath: path.join(dirPath, "meta.json"),
+    key,
+    urlSha256: sha256(url),
+  };
+}
+
+export function countLines(markdown: string): number {
+  return markdown.split(/\r?\n/).length;
+}
+
+export function paginate(
+  markdown: string,
+  offset: number,
+  limit: number,
+  maxBytes: number = DEFAULT_MAX_BYTES,
+): Pagination {
+  const lines = markdown.split(/\r?\n/);
+  const totalLines = lines.length;
+  const startIndex = Math.max(0, offset - 1);
+  if (startIndex >= totalLines) {
+    return {
+      selected: "",
+      totalLines,
+      shownStart: totalLines + 1,
+      shownEnd: totalLines,
+      truncated: false,
+      shownBytes: 0,
+      totalBytes: 0,
+    };
+  }
+  const endIndex = Math.min(totalLines, startIndex + limit);
+  const windowText = lines.slice(startIndex, endIndex).join("\n");
+  // Truncate the selected window by bytes here so nextOffset reflects the lines
+  // actually emitted. Computing nextOffset from the full line window would
+  // permanently skip any tail lines dropped by byte truncation.
+  const truncation = truncateHead(windowText, { maxBytes });
+  // Math.max(1, ...) guarantees forward progress even when a single oversized
+  // line yields zero output lines, avoiding a nextOffset === offset loop.
+  const shownLineCount = truncation.truncated
+    ? Math.max(1, truncation.outputLines)
+    : endIndex - startIndex;
+  const shownEnd = startIndex + shownLineCount;
+  return {
+    selected: truncation.content,
+    totalLines,
+    shownStart: Math.min(offset, totalLines),
+    shownEnd,
+    nextOffset: shownEnd < totalLines ? shownEnd + 1 : undefined,
+    truncated: truncation.truncated,
+    shownBytes: truncation.outputBytes,
+    totalBytes: truncation.totalBytes,
+  };
+}
+
+export async function loadCached(
+  url: string,
+): Promise<CachedDocument | undefined> {
+  const paths = cachePaths(url);
+  try {
+    return await loadCachedFromPaths(url, paths);
+  } catch (error) {
+    if (!isNotFoundError(error)) {
+      console.warn(
+        `[read-page] Ignoring corrupt cache for ${url}: ${errorMessage(error)}`,
+      );
+    }
+    return undefined;
+  }
+}
+
+export async function loadCachedFromPaths(
+  url: string,
+  paths: { mdPath: string; metaPath: string },
+): Promise<CachedDocument> {
+  const [markdown, metaRaw] = await Promise.all([
+    readFile(paths.mdPath, "utf8"),
+    readFile(paths.metaPath, "utf8"),
+  ]);
+  const meta = JSON.parse(metaRaw) as CacheMeta;
+  if (meta.url !== url) throw new Error("Cache URL mismatch");
+  if (meta.content_sha256 !== sha256(markdown))
+    throw new Error("Cache checksum mismatch");
+  if (!Number.isFinite(Date.parse(meta.expires_at)))
+    throw new Error("Invalid cache expires_at");
+  const fresh = Date.now() < Date.parse(meta.expires_at);
+  return { markdown, meta, fresh };
+}
+
+function isNotFoundError(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    (error as { code?: unknown }).code === "ENOENT"
+  );
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+async function writeFileAtomic(
+  filePath: string,
+  content: string,
+): Promise<void> {
+  const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+  await writeFile(tmpPath, content, "utf8");
+  await rename(tmpPath, filePath);
+}
+
+export async function saveCached(params: {
+  normalized: NormalizedUrl;
+  finalUrl: string;
+  markdown: string;
+  extractor: "defuddle";
+  extraction: string;
+  parseMode: string;
+  userAction: boolean;
+  confidence: ConfidenceReport;
+  metadata: PageMetadata;
+  browserProfile?: "persistent" | "temporary";
+}): Promise<CacheMeta> {
+  const { dirPath, mdPath, metaPath, key, urlSha256 } = cachePaths(
+    params.normalized.url,
+  );
+  await mkdir(dirPath, { recursive: true });
+
+  const now = Date.now();
+  const ttlDays = params.userAction ? USER_ACTION_TTL_DAYS : DEFAULT_TTL_DAYS;
+  const ttlMs = ttlDays * 24 * 60 * 60 * 1000;
+
+  const meta: CacheMeta = {
+    version: 1,
+    input_url: params.normalized.inputUrl,
+    url: params.normalized.url,
+    final_url: params.finalUrl,
+    cache_key: key,
+    url_sha256: urlSha256,
+    normalization: params.normalized.normalization,
+    source: "browser",
+    extractor: params.extractor,
+    extraction: params.extraction,
+    parse_mode: params.parseMode,
+    browser_profile: params.browserProfile,
+    user_action: params.userAction,
+    confidence: params.confidence,
+    metadata: params.metadata,
+    fetched_at: new Date(now).toISOString(),
+    expires_at: new Date(now + ttlMs).toISOString(),
+    ttl_days: ttlDays,
+    content_sha256: sha256(params.markdown),
+    chars: params.markdown.length,
+    lines: countLines(params.markdown),
+  };
+
+  await writeFileAtomic(mdPath, params.markdown);
+  await writeFileAtomic(metaPath, `${JSON.stringify(meta, null, 2)}\n`);
+  return meta;
+}

package/src/security/url-policy.ts

@@ -0,0 +1,345 @@
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+
+export type UrlNormalization = {
+  strip_fragment: boolean;
+  strip_query: boolean;
+  strip_trailing_slash: boolean;
+};
+
+export type NormalizedUrl = {
+  inputUrl: string;
+  url: string;
+  normalization: UrlNormalization;
+};
+
+const dnsPolicyChecks = new Map<string, Promise<void>>();
+const MAX_FETCH_REDIRECTS = 20;
+
+export function normalizeHttpUrl(
+  input: string,
+  options: { preserveQuery: boolean },
+): NormalizedUrl {
+  const parsed = parseHttpUrl(input);
+  const inputUrl = input.trim();
+
+  parsed.hostname = parsed.hostname.toLowerCase();
+  enforceHostPolicy(parsed.hostname);
+  parsed.hash = "";
+
+  const stripQuery = !options.preserveQuery;
+  if (stripQuery) parsed.search = "";
+
+  const stripTrailingSlash =
+    parsed.pathname !== "/" && parsed.pathname.endsWith("/");
+  if (stripTrailingSlash)
+    parsed.pathname = parsed.pathname.replace(/\/+$/g, "");
+
+  return {
+    inputUrl,
+    url: parsed.toString(),
+    normalization: {
+      strip_fragment: true,
+      strip_query: stripQuery,
+      strip_trailing_slash: stripTrailingSlash,
+    },
+  };
+}
+
+export async function assertHttpUrlAllowed(url: string): Promise<void> {
+  const parsed = parseHttpUrl(url);
+  enforceHostPolicy(parsed.hostname);
+
+  if (allowsPrivateNetwork()) return;
+  const host = normalizeHost(parsed.hostname);
+  if (isIP(host)) return;
+
+  const currentCheck = dnsPolicyChecks.get(host);
+  if (currentCheck) {
+    await currentCheck;
+    return;
+  }
+
+  let check!: Promise<void>;
+  check = enforceDnsPolicy(host).finally(() => {
+    if (dnsPolicyChecks.get(host) === check) dnsPolicyChecks.delete(host);
+  });
+  dnsPolicyChecks.set(host, check);
+  await check;
+}
+
+export function isHttpLikeUrl(url: string): boolean {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+
+export function createPolicyFetch(
+  delegate: typeof globalThis.fetch = globalThis.fetch,
+): typeof globalThis.fetch {
+  return async (input, init) => {
+    const redirectMode = fetchRedirectMode(input, init);
+    if (redirectMode !== "follow") {
+      await assertHttpUrlAllowed(fetchInputUrl(input));
+      return delegate(input, init);
+    }
+
+    let nextInput: RequestInfo | URL = input;
+    let nextInit: RequestInit = { ...init, redirect: "manual" };
+
+    for (let redirects = 0; redirects <= MAX_FETCH_REDIRECTS; redirects += 1) {
+      const currentUrl = fetchInputUrl(nextInput);
+      await assertHttpUrlAllowed(currentUrl);
+      const response = await delegate(nextInput, nextInit);
+      if (!isRedirectResponse(response)) return response;
+
+      const location = response.headers.get("location");
+      if (!location) return response;
+      if (redirects === MAX_FETCH_REDIRECTS) {
+        throw new Error("Fetch redirect limit exceeded");
+      }
+
+      const currentInput = nextInput;
+      nextInput = new URL(location, currentUrl).href;
+      nextInit = nextRedirectInit(currentInput, nextInit, response.status);
+    }
+
+    throw new Error("Fetch redirect limit exceeded");
+  };
+}
+
+function fetchInputUrl(input: RequestInfo | URL): string {
+  if (typeof input === "string") return input;
+  if (input instanceof URL) return input.href;
+  if (typeof input === "object" && input !== null && "url" in input) {
+    const url = input.url;
+    if (typeof url === "string") return url;
+  }
+  throw new Error("Unable to determine fetch request URL");
+}
+
+function fetchRedirectMode(
+  input: RequestInfo | URL,
+  init: RequestInit | undefined,
+): RequestRedirect {
+  if (init?.redirect) return init.redirect;
+  if (typeof input === "object" && input !== null && "redirect" in input) {
+    const redirect = input.redirect;
+    if (
+      redirect === "error" ||
+      redirect === "follow" ||
+      redirect === "manual"
+    ) {
+      return redirect;
+    }
+  }
+  return "follow";
+}
+
+function fetchMethod(input: RequestInfo | URL, init: RequestInit): string {
+  if (init.method) return init.method.toUpperCase();
+  if (typeof input === "object" && input !== null && "method" in input) {
+    const method = input.method;
+    if (typeof method === "string") return method.toUpperCase();
+  }
+  return "GET";
+}
+
+function nextRedirectInit(
+  input: RequestInfo | URL,
+  init: RequestInit,
+  status: number,
+): RequestInit {
+  const method = fetchMethod(input, init);
+  if (
+    status !== 303 &&
+    !((status === 301 || status === 302) && method === "POST")
+  ) {
+    return init;
+  }
+  return { ...init, method: "GET", body: undefined };
+}
+
+function isRedirectResponse(response: Response): boolean {
+  return [301, 302, 303, 307, 308].includes(response.status);
+}
+
+function parseHttpUrl(input: string): URL {
+  let parsed: URL;
+
+  try {
+    parsed = new URL(input.trim());
+  } catch {
+    throw new Error(`Invalid URL: ${input}`);
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error(
+      `Only http:// and https:// URLs are supported. Refusing URL: ${input}`,
+    );
+  }
+
+  return parsed;
+}
+
+async function enforceDnsPolicy(host: string): Promise<void> {
+  let records: Array<{ address: string }>;
+  try {
+    records = await lookup(host, { all: true, verbatim: true });
+  } catch (error) {
+    throw new Error(
+      `Failed to resolve hostname ${host}: ${errorMessage(error)}`,
+    );
+  }
+
+  for (const record of records) {
+    if (isPrivateIp(record.address)) {
+      throw new Error(
+        `Refusing hostname ${host} because DNS resolved to private/local address ${record.address}. Set READ_PAGE_ALLOW_PRIVATE_NETWORK=1 to allow it explicitly.`,
+      );
+    }
+  }
+}
+
+function enforceHostPolicy(hostname: string): void {
+  if (allowsPrivateNetwork()) return;
+
+  const host = normalizeHost(hostname);
+  if (
+    host === "localhost" ||
+    host.endsWith(".localhost") ||
+    host.endsWith(".local") ||
+    host.endsWith(".internal")
+  ) {
+    throw new Error(
+      `Refusing private/local hostname: ${hostname}. Set READ_PAGE_ALLOW_PRIVATE_NETWORK=1 to allow it explicitly.`,
+    );
+  }
+
+  if (isIP(host) && isPrivateIp(host)) {
+    throw new Error(
+      `Refusing private/local IP address: ${hostname}. Set READ_PAGE_ALLOW_PRIVATE_NETWORK=1 to allow it explicitly.`,
+    );
+  }
+}
+
+function isPrivateIp(ip: string): boolean {
+  const ipVersion = isIP(ip);
+  if (ipVersion === 4) return isPrivateIPv4(ip);
+  if (ipVersion === 6) return isPrivateIPv6(ip);
+  return true;
+}
+
+function isPrivateIPv4(ip: string): boolean {
+  const parts = ip.split(".").map((part) => Number.parseInt(part, 10));
+  if (
+    parts.length !== 4 ||
+    parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)
+  ) {
+    return true;
+  }
+
+  const [a, b] = parts;
+  return (
+    a === 0 ||
+    a === 10 ||
+    a === 127 ||
+    (a === 100 && b >= 64 && b <= 127) ||
+    (a === 169 && b === 254) ||
+    (a === 172 && b >= 16 && b <= 31) ||
+    (a === 192 && b === 168) ||
+    (a === 192 && b === 0) ||
+    (a === 198 && (b === 18 || b === 19)) ||
+    a >= 224
+  );
+}
+
+function isPrivateIPv6(ip: string): boolean {
+  const normalized = normalizeHost(ip).split("%")[0] || "";
+  const mappedIpv4 = ipv4FromMappedIPv6(normalized);
+  if (mappedIpv4) return isPrivateIPv4(mappedIpv4);
+
+  const words = parseIPv6Words(normalized);
+  if (!words) return true;
+  const first = words[0] ?? 0;
+  return (
+    normalized === "::1" ||
+    normalized === "::" ||
+    first === 0 ||
+    (first & 0xffc0) === 0xfe80 ||
+    (first & 0xfe00) === 0xfc00 ||
+    (first & 0xff00) === 0xff00
+  );
+}
+
+function ipv4FromMappedIPv6(ip: string): string | undefined {
+  const words = parseIPv6Words(ip);
+  if (!words) return undefined;
+  if (words.slice(0, 5).some((word) => word !== 0) || words[5] !== 0xffff) {
+    return undefined;
+  }
+
+  const high = words[6] ?? 0;
+  const low = words[7] ?? 0;
+  return [high >> 8, high & 0xff, low >> 8, low & 0xff].join(".");
+}
+
+function parseIPv6Words(ip: string): number[] | undefined {
+  let input = ip.toLowerCase();
+  if (input.includes(".")) {
+    const lastColon = input.lastIndexOf(":");
+    if (lastColon === -1) return undefined;
+    const ipv4 = input.slice(lastColon + 1);
+    if (isIP(ipv4) !== 4) return undefined;
+    const octets = ipv4.split(".").map((part) => Number.parseInt(part, 10));
+    const [a, b, c, d] = octets;
+    if (
+      a === undefined ||
+      b === undefined ||
+      c === undefined ||
+      d === undefined
+    ) {
+      return undefined;
+    }
+    input = `${input.slice(0, lastColon + 1)}${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}`;
+  }
+
+  const compressionParts = input.split("::");
+  if (compressionParts.length > 2) return undefined;
+
+  const left = parseIPv6Side(compressionParts[0] ?? "");
+  const right = parseIPv6Side(compressionParts[1] ?? "");
+  if (!left || !right) return undefined;
+
+  if (compressionParts.length === 1) {
+    return left.length === 8 ? left : undefined;
+  }
+
+  const missing = 8 - left.length - right.length;
+  if (missing < 1) return undefined;
+  return [...left, ...Array.from({ length: missing }, () => 0), ...right];
+}
+
+function parseIPv6Side(input: string): number[] | undefined {
+  if (!input) return [];
+  const words = input.split(":").map((part) => {
+    if (!/^[0-9a-f]{1,4}$/.test(part)) return Number.NaN;
+    return Number.parseInt(part, 16);
+  });
+  return words.every(Number.isFinite) ? words : undefined;
+}
+
+function normalizeHost(hostname: string): string {
+  return hostname.replace(/^\[|\]$/g, "").toLowerCase();
+}
+
+function allowsPrivateNetwork(): boolean {
+  return process.env.READ_PAGE_ALLOW_PRIVATE_NETWORK === "1";
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}