pim-agent 0.0.1 → 0.9.0

package/packages/core/src/extensions/web-fetch/fetch.ts

@@ -0,0 +1,265 @@
+import { Errors } from "../../shared/Errors";
+import { isIP } from "node:net";
+import type { HttpFetch } from "../../shared/Http";
+import { Lines } from "../../shared/Lines";
+import { OutputBudget } from "../../shared/OutputBudget";
+import { SpillCache } from "../../shared/SpillCache";
+import { fetchImage, type WebFetchImageOutcome } from "./image";
+import type { JinaReaderClient } from "./JinaReaderClient";
+import {
+  WEB_FETCH_INLINE_BYTES,
+  type WebFetchFormat,
+  type WebFetchResolvedFormat,
+} from "./schema";
+import type { WebViewFetchClient } from "./WebViewFetchClient";
+
+export type WebFetchPage = {
+  readonly title: string;
+  readonly url: string;
+  readonly content: string;
+};
+
+export type WebFetchPageOutcome = {
+  readonly kind: "page";
+  readonly text: string;
+  readonly title: string;
+  readonly url: string;
+  readonly format: WebFetchResolvedFormat;
+  readonly returnedBytes: number;
+  readonly totalBytes: number;
+  readonly truncated: boolean;
+  readonly path: string | null;
+};
+
+export type WebFetchOutcome = WebFetchPageOutcome | WebFetchImageOutcome;
+
+export function truncationFooter(
+  returnedBytes: number,
+  totalBytes: number,
+  spillPath: string | null,
+  nextStart: number
+): string {
+  const base = `[web_fetch tool: showing first ${returnedBytes} bytes of ${totalBytes}`;
+  if (spillPath) {
+    return `${base}; use read with path=${spillPath} and start=${nextStart} for the rest.]`;
+  }
+  return `${base}; full content unavailable.]`;
+}
+
+export function validatePublicUrl(value: string): string {
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`URL must be a valid public HTTP or HTTPS URL: ${value}`);
+  }
+
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`URL must use http:// or https://: ${value}`);
+  }
+
+  if (url.username.length > 0 || url.password.length > 0) {
+    throw new Error("URL must not contain embedded credentials.");
+  }
+
+  if (!isPublicHostname(url.hostname)) {
+    throw new Error(
+      `URL must use a public hostname or IP address (no localhost, .local, or RFC1918 ranges): ${url.hostname}`
+    );
+  }
+
+  return url.href;
+}
+
+export async function formatOutcome(
+  page: WebFetchPage,
+  format: WebFetchResolvedFormat
+): Promise<WebFetchPageOutcome> {
+  const { body, returnedBytes, totalBytes, truncated } =
+    OutputBudget.truncateUtf8(page.content, WEB_FETCH_INLINE_BYTES);
+  const path = truncated
+    ? await SpillCache.write(
+        "fetch",
+        format === "html" ? "html" : "md",
+        page.content
+      )
+    : null;
+
+  const lines = [
+    `title: ${page.title}`,
+    `url: ${page.url}`,
+    `format: ${format}`,
+    "content:",
+    body,
+  ];
+
+  if (truncated) {
+    const nextStart = Lines.continuationLine(body);
+    lines.push(
+      "",
+      truncationFooter(returnedBytes, totalBytes, path, nextStart)
+    );
+  }
+
+  return {
+    kind: "page",
+    text: lines.join("\n"),
+    title: page.title,
+    url: page.url,
+    format,
+    returnedBytes,
+    totalBytes,
+    truncated,
+    path,
+  };
+}
+
+export type ExecuteFetchInput = {
+  readonly jina: JinaReaderClient;
+  readonly webView: WebViewFetchClient;
+  readonly url: string;
+  readonly format: WebFetchFormat;
+  readonly signal?: AbortSignal;
+  /** Only the image probe takes it; the readers own their own transports. */
+  readonly fetch?: HttpFetch;
+};
+
+export async function executeFetch(
+  input: ExecuteFetchInput
+): Promise<WebFetchOutcome> {
+  const { jina, webView, url, format, signal } = input;
+  const fetchInput = {
+    url,
+    ...(signal === undefined ? {} : { signal }),
+  };
+
+  const image = await fetchImage({
+    ...fetchInput,
+    fetch: input.fetch,
+  });
+
+  if (image !== null) {
+    return { kind: "image", url, image };
+  }
+
+  if (format === "html") {
+    const page = await webView.fetchHtml(fetchInput);
+    return formatOutcome(page, "html");
+  }
+
+  const sources: readonly WebFetchSource[] = [
+    () => jina.fetchUrl(fetchInput),
+    () => webView.fetchMarkdown(fetchInput),
+  ];
+  let lastError: unknown;
+
+  for (const source of sources) {
+    try {
+      return await formatOutcome(await source(), "markdown");
+    } catch (error) {
+      if (signal?.aborted) {
+        throw error;
+      }
+      lastError = error;
+    }
+  }
+
+  throw new Error(`Failed to fetch: ${Errors.describe(lastError)}`);
+}
+
+type WebFetchSource = () => Promise<WebFetchPage>;
+
+function isPublicHostname(hostname: string): boolean {
+  const normalized = normalizeHostname(hostname);
+
+  if (normalized === "localhost" || normalized.endsWith(".local")) {
+    return false;
+  }
+
+  const version = isIP(normalized);
+
+  if (version === 4) {
+    return isPublicIpv4(normalized);
+  }
+
+  if (version === 6) {
+    return isPublicIpv6(normalized);
+  }
+
+  return normalized.length > 0;
+}
+
+function normalizeHostname(hostname: string): string {
+  const lower = hostname.toLowerCase().replace(/\.+$/u, "");
+
+  if (lower.startsWith("[") && lower.endsWith("]")) {
+    return lower.slice(1, -1);
+  }
+
+  return lower;
+}
+
+function isPublicIpv4(ip: string): boolean {
+  const octets = ip.split(".").map((octet) => Number(octet));
+  const [a = 0, b = 0, c = 0] = octets;
+
+  if (a === 0 || a === 10 || a === 127) {
+    return false;
+  }
+  if (a === 169 && b === 254) {
+    return false;
+  }
+  if (a === 172 && b >= 16 && b <= 31) {
+    return false;
+  }
+  if (a === 192 && b === 168) {
+    return false;
+  }
+  if (a === 100 && b >= 64 && b <= 127) {
+    return false;
+  }
+  if (a === 192 && b === 0 && c === 0) {
+    return false;
+  }
+  if (a === 198 && (b === 18 || b === 19)) {
+    return false;
+  }
+
+  return a < 224;
+}
+
+function isPublicIpv6(ip: string): boolean {
+  const embedded = readEmbeddedIpv4(ip);
+
+  if (embedded !== undefined && !isPublicIpv4(embedded)) {
+    return false;
+  }
+
+  if (ip === "::" || ip === "::1") {
+    return false;
+  }
+
+  const first = readFirstIpv6Segment(ip);
+
+  if ((first & 0xfe00) === 0xfc00) {
+    return false;
+  }
+  if ((first & 0xffc0) === 0xfe80) {
+    return false;
+  }
+
+  return (first & 0xff00) !== 0xff00;
+}
+
+function readEmbeddedIpv4(ip: string): string | undefined {
+  const match = /(?:^|:)(\d{1,3}(?:\.\d{1,3}){3})$/u.exec(ip);
+  return match?.[1];
+}
+
+function readFirstIpv6Segment(ip: string): number {
+  const first =
+    ip
+      .split(":")
+      .find((segment) => segment.length > 0 && !segment.includes(".")) ?? "0";
+  return Number.parseInt(first, 16);
+}

package/packages/core/src/extensions/web-fetch/image.ts

@@ -0,0 +1,165 @@
+import type { AgentToolResult } from "@earendil-works/pi-coding-agent";
+import { Format } from "../../shared/Format";
+import type { HttpFetch } from "../../shared/Http";
+import { Images, type NormalisedImage } from "../../shared/Images";
+import type { WebFetchImageDetails } from "./schema";
+
+export type WebFetchImageOutcome = {
+  readonly kind: "image";
+  readonly url: string;
+  readonly image: NormalisedImage;
+};
+
+export type ImageFetchInput = {
+  readonly url: string;
+  readonly signal?: AbortSignal;
+  readonly fetch?: HttpFetch;
+};
+
+const TIMEOUT_MS = 20_000;
+
+const IMAGE_CONTENT_TYPE = /^image\//iu;
+
+/** The picture the URL really serves, or null for anything the page path should handle. */
+export async function fetchImage(
+  input: ImageFetchInput
+): Promise<NormalisedImage | null> {
+  const response = await declaredImage(input);
+
+  if (response === null) {
+    return null;
+  }
+
+  const bytes = await readCapped(response, input.url);
+  return bytes === null ? null : Images.normalise(bytes, input.url);
+}
+
+export function imageContent(
+  image: NormalisedImage
+): AgentToolResult<unknown>["content"] {
+  const footer =
+    image.cachePath === null
+      ? []
+      : [
+          {
+            type: "text" as const,
+            text: `[web_fetch tool: image saved to ${image.cachePath}]`,
+          },
+        ];
+  return [...Images.contentOf(image, Images.noteOf(image)), ...footer];
+}
+
+/** The fetch succeeded and the intent was ambiguous, so the model is told what it got, not refused. */
+export function noVisionNote(url: string, image: NormalisedImage): string {
+  const saved = image.cachePath === null ? "" : `, saved to ${image.cachePath}`;
+  return Images.noVisionNote(
+    "web_fetch",
+    `${url} is a ${image.width}x${image.height} ${Images.extensionOf(image.mimeType)}${saved}`
+  );
+}
+
+export function imageDetails(
+  url: string,
+  image: NormalisedImage,
+  withheld: boolean
+): WebFetchImageDetails {
+  return {
+    kind: "image",
+    url,
+    ...Images.detailsOf(image),
+    path: image.cachePath,
+    withheld,
+  };
+}
+
+/** A response whose own headers claim a picture; the bytes still have to agree. */
+async function declaredImage(input: ImageFetchInput): Promise<Response | null> {
+  let response: Response;
+
+  try {
+    response = await (input.fetch ?? fetch)(input.url, {
+      headers: { Accept: "image/*,*/*;q=0.8" },
+      signal: deadline(input.signal),
+    });
+  } catch (error) {
+    if (input.signal?.aborted) {
+      throw error;
+    }
+    return null;
+  }
+
+  const contentType = response.headers.get("content-type") ?? "";
+
+  if (!response.ok || !IMAGE_CONTENT_TYPE.test(contentType)) {
+    await discard(response);
+    return null;
+  }
+
+  if (
+    Number(response.headers.get("content-length")) > Images.MAX_SOURCE_BYTES
+  ) {
+    await discard(response);
+    throw oversize(input.url);
+  }
+
+  return response;
+}
+
+/** Stops at the cap and at the first chunk that is not a picture, so neither is ever buffered whole. */
+async function readCapped(
+  response: Response,
+  url: string
+): Promise<Uint8Array | null> {
+  const reader = response.body?.getReader();
+
+  if (reader === undefined) {
+    return null;
+  }
+
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  let sniffed = false;
+
+  while (true) {
+    const { done, value } = await reader.read();
+
+    if (done) {
+      break;
+    }
+
+    chunks.push(value);
+    total += value.byteLength;
+
+    if (total > Images.MAX_SOURCE_BYTES) {
+      await reader.cancel();
+      throw oversize(url);
+    }
+
+    if (!sniffed && total >= Images.SNIFF_BYTES) {
+      if (Images.sniff(Buffer.concat(chunks)) === null) {
+        await reader.cancel();
+        return null;
+      }
+      sniffed = true;
+    }
+  }
+
+  return sniffed ? Buffer.concat(chunks) : null;
+}
+
+function oversize(url: string): Error {
+  return new Error(
+    `Image at ${url} is over the ${Format.bytes(Images.MAX_SOURCE_BYTES)} fetch cap. Download and shrink it with bash: curl -sL ${url} -o /tmp/image && magick /tmp/image -resize 2000x2000 /tmp/small.png`
+  );
+}
+
+function deadline(signal: AbortSignal | undefined): AbortSignal {
+  const timeout = AbortSignal.timeout(TIMEOUT_MS);
+  return signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
+}
+
+async function discard(response: Response): Promise<void> {
+  try {
+    await response.body?.cancel();
+  } catch {}
+}

package/packages/core/src/extensions/web-fetch/index.ts

@@ -0,0 +1,83 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Images } from "../../shared/Images";
+import { PimSettings } from "../../shared/PimSettings";
+import { SpillCache } from "../../shared/SpillCache";
+import { Tools } from "../../shared/Tools";
+import { executeFetch, validatePublicUrl } from "./fetch";
+import { imageContent, imageDetails, noVisionNote } from "./image";
+import { JinaReaderClient } from "./JinaReaderClient";
+import { webFetchView } from "./render";
+import {
+  type WebFetchDetails,
+  type WebFetchInput,
+  webFetchSchema,
+} from "./schema";
+import { WebViewFetchClient } from "./WebViewFetchClient";
+
+async function createJina(): Promise<JinaReaderClient> {
+  const apiKey = await PimSettings.getJinaApiKey();
+  return new JinaReaderClient(apiKey ? { apiKey } : {});
+}
+
+export default function (pi: ExtensionAPI): void {
+  SpillCache.installSweeper();
+
+  let jinaPromise: Promise<JinaReaderClient> | undefined;
+  const getJina = () => (jinaPromise ??= createJina());
+  const webView = new WebViewFetchClient();
+
+  Tools.register<typeof webFetchSchema, WebFetchDetails>(pi, {
+    name: "web_fetch",
+    label: "web_fetch",
+    description:
+      "Fetch a web page as markdown or HTML. " +
+      "A URL serving an image (png, jpeg, gif, webp) returns the picture itself, saved to the cache; the format parameter does not apply to it.",
+    parameters: webFetchSchema,
+    renderShell: "self",
+    effect: { kind: "readOnly" },
+    executionMode: "parallel",
+    async execute(_id, params, signal, _onUpdate, ctx) {
+      const { url, format } = params as WebFetchInput;
+
+      if (signal?.aborted) {
+        throw new Error("Web fetch aborted before execution.");
+      }
+
+      const safeUrl = validatePublicUrl(url);
+
+      const jina = await getJina();
+      const outcome = await executeFetch({
+        jina,
+        webView,
+        url: safeUrl,
+        format: format ?? "markdown",
+        ...(signal === undefined ? {} : { signal }),
+      });
+
+      if (outcome.kind === "image") {
+        const withheld = !Images.canSee(ctx.model);
+        return {
+          content: withheld
+            ? [{ type: "text", text: noVisionNote(outcome.url, outcome.image) }]
+            : imageContent(outcome.image),
+          details: imageDetails(outcome.url, outcome.image, withheld),
+        };
+      }
+
+      return {
+        content: [{ type: "text", text: outcome.text }],
+        details: {
+          kind: "page",
+          url: outcome.url,
+          title: outcome.title,
+          format: outcome.format,
+          returnedBytes: outcome.returnedBytes,
+          totalBytes: outcome.totalBytes,
+          truncated: outcome.truncated,
+          path: outcome.path,
+        },
+      };
+    },
+    toViewModel: webFetchView,
+  });
+}

package/packages/core/src/extensions/web-fetch/render.ts

@@ -0,0 +1,100 @@
+import { Format } from "../../shared/Format";
+import { Images } from "../../shared/Images";
+import { Renderer } from "../../shared/Renderer";
+import type { ToolViewInput } from "../../shared/Tools";
+import { Painting } from "../../view/Painting";
+import type { ToolView, ViewBlock } from "../../view/ViewBlock";
+import type {
+  WebFetchDetails,
+  WebFetchFormat,
+  WebFetchImageDetails,
+  WebFetchInput,
+  WebFetchResolvedFormat,
+  webFetchSchema,
+} from "./schema";
+
+type WebFetchViewInput = ToolViewInput<typeof webFetchSchema, WebFetchDetails>;
+
+type TitleOutcome = {
+  readonly format: WebFetchResolvedFormat;
+  readonly totalBytes: number;
+};
+
+export function webFetchView({ args, result }: WebFetchViewInput): ToolView {
+  const input = (args ?? {}) as Partial<WebFetchInput>;
+  const image = imageDetailsOf(result?.details);
+  const url = image?.url ?? input.url ?? "...";
+  return {
+    label: "Web Fetch",
+    icon: "globe",
+    title: [
+      { kind: "text", text: url },
+      {
+        kind: "text",
+        tone: "muted",
+        text: image
+          ? imageDetail(image)
+          : formatDetail(input.format, titleOutcome(result)),
+      },
+    ],
+    body: image ? imageBody(image, url) : formatBody(result),
+  };
+}
+
+/** Legacy sessions predate the tag, so anything untagged reads as a page. */
+function imageDetailsOf(
+  details: WebFetchDetails | undefined
+): WebFetchImageDetails | undefined {
+  return details?.kind === "image" ? details : undefined;
+}
+
+function imageBody(
+  details: WebFetchImageDetails,
+  url: string
+): readonly ViewBlock[] {
+  return Painting.imageBlocks(
+    details,
+    url,
+    details.withheld
+      ? [["not sent", "the current model has no vision input"]]
+      : []
+  );
+}
+
+function imageDetail(details: WebFetchImageDetails): string {
+  return `${Format.bytesCompact(details.bytes)} ${Images.extensionOf(details.mimeType).toUpperCase()}`;
+}
+
+function titleOutcome(
+  result: WebFetchViewInput["result"]
+): TitleOutcome | undefined {
+  const details = result?.details;
+  if (
+    details === undefined ||
+    details.kind === "image" ||
+    details.format === undefined ||
+    typeof details.totalBytes !== "number"
+  ) {
+    return undefined;
+  }
+  return { format: details.format, totalBytes: details.totalBytes };
+}
+
+function formatBody(result: WebFetchViewInput["result"]): readonly ViewBlock[] {
+  const text = Renderer.firstText(result);
+  return text === "" ? [] : [{ kind: "text", text }];
+}
+
+function formatDetail(
+  format: WebFetchFormat | undefined,
+  outcome: TitleOutcome | undefined
+): string {
+  const label = formatLabel(outcome?.format ?? format ?? "markdown");
+  return outcome === undefined
+    ? label
+    : `${Format.bytesCompact(outcome.totalBytes)} ${label}`;
+}
+
+function formatLabel(format: WebFetchResolvedFormat): string {
+  return format === "html" ? "HTML" : "Markdown";
+}

package/packages/core/src/extensions/web-fetch/schema.ts

@@ -0,0 +1,46 @@
+import { StringEnum } from "@earendil-works/pi-ai";
+import { type Static, Type } from "typebox";
+import type { ImageDetails } from "../../shared/Images";
+
+export const WEB_FETCH_INLINE_BYTES = 32 * 1024;
+
+export const FETCH_FORMATS = ["markdown", "html"] as const;
+export type WebFetchFormat = (typeof FETCH_FORMATS)[number];
+export type WebFetchResolvedFormat = WebFetchFormat;
+
+export const webFetchSchema = Type.Object({
+  url: Type.String({
+    minLength: 1,
+    description: "Must be a public http(s) URL.",
+  }),
+  format: Type.Optional(
+    StringEnum(FETCH_FORMATS, {
+      description:
+        "`markdown`: used by default. `html`: use only when raw source is required.",
+    })
+  ),
+});
+
+export type WebFetchInput = Static<typeof webFetchSchema>;
+
+export type WebFetchPageDetails = {
+  readonly kind: "page";
+  readonly url: string;
+  readonly title: string;
+  readonly format: WebFetchResolvedFormat;
+  readonly returnedBytes: number;
+  readonly totalBytes: number;
+  readonly truncated: boolean;
+  readonly path: string | null;
+};
+
+export type WebFetchImageDetails = ImageDetails & {
+  readonly kind: "image";
+  readonly url: string;
+  /** `~/.pim/cache/img-<sha256>.<ext>`, or null when the cache write failed. */
+  readonly path: string | null;
+  /** Cached and shown here, but never sent: the current model has no vision input. */
+  readonly withheld: boolean;
+};
+
+export type WebFetchDetails = WebFetchPageDetails | WebFetchImageDetails;