pi-central 0.6.0 → 0.7.0

package/README.md

@@ -124,12 +124,14 @@ Stated plainly, because a dashboard that overstates itself is worse than one tha
 **Works:** multiple sandboxed sessions in different directories with verified isolation; live transcripts
 reconstructed from Pi's delta-only stream; the approval dock answerable for any session; a bounded tool
 trace with full traversal; a read-only file tree and image viewer confined to the session's directory;
+images shown where they belong in the conversation, including the ones a tool read, and screenshots pasted or
+dropped straight into the composer;
 history searchable by content, with a clear refusal when a session's directory is gone; a session roster
 that survives a restart; agent-scoped credentials with the host's own credentials denied, and commits
 signed as the agent when the store is configured to sign.
 
-**Not yet:** no file editing from the browser, no update checking, no export. Each is a tracked issue
-rather than a wish.
+**Not yet:** no file editing from the browser, no update checking, no export. The image size caps are derived from the
+transport's frame limit rather than tuned against real reading. Each is a tracked issue rather than a wish.
 
 ## Working on this project
 

package/bin/pi-central.js

@@ -25,7 +25,7 @@ function isBlockingDialog(record) {
 }
 
 // packages/protocol/src/envelopes.ts
-var PROTOCOL_VERSION = 4;
+var PROTOCOL_VERSION = 5;
 
 // packages/protocol/src/own-installation.ts
 var PIPELINE_MARKERS = ["packages/orchestrator", "packages/ui", "packages/protocol"];
@@ -1479,6 +1479,7 @@ var blockingDialogMethods = [...BLOCKING_DIALOG_METHODS];
 var fireAndForgetUiMethods = [...FIRE_AND_FORGET_UI_METHODS];
 
 // packages/orchestrator/src/supervisor.ts
+var EMPTY_PROMPT_WITH_IMAGES = "See the attached image.";
 var Supervisor = class extends EventEmitter2 {
   #config;
   #sessions = /* @__PURE__ */ new Map();
@@ -1848,22 +1849,32 @@ ${rpc.stderrTail}` : ""}`;
     this.#patch(id);
   }
   /**
-   * Send a prompt. The command shape is closed; the message is the only free value.
+   * Send a prompt. The command shape is closed; the message and its images are the only free values.
    *
    * `behavior` decides what a message means mid-run, and it is the caller's choice rather than
    * this method's: Pi REJECTS a streaming `prompt` with no `streamingBehavior` (`"Agent is
    * already processing. Specify streamingBehavior ('steer' or 'followUp')"`), so defaulting it
    * silently would decide for the user something the composer is there to ask (issue #3). When
    * idle there is nothing to steer into, so the flag is dropped rather than sent.
+   *
+   * `images` arrive ALREADY VALIDATED (`prompt-images.ts`) and are mapped to Pi's shape here — the one
+   * place a control becomes Pi's stdin, which is the right place for the spelling to change. Pi wants
+   * `{ type: "image", data, mimeType }`; the wire carries `{ mediaType, data }` because `type` is the
+   * discriminant of every record on that wire and reusing it inside a payload makes a log ambiguous.
    */
-  async prompt(id, message, behavior) {
+  async prompt(id, message, behavior, images = []) {
     const rpc = this.#processes.get(id);
     const session = this.#sessions.get(id);
     if (!rpc || !session) throw new Error(`no such session: ${id}`);
+    const text = message.trim().length === 0 && images.length > 0 ? EMPTY_PROMPT_WITH_IMAGES : message;
+    const withImages = images.length > 0 ? { images: images.map((image) => ({ type: "image", data: image.data, mimeType: image.mediaType })) } : {};
     if (session.state === "streaming") {
-      await rpc.send({ type: "prompt", message, streamingBehavior: behavior ?? "followUp" }, 6e4);
+      await rpc.send(
+        { type: "prompt", message: text, streamingBehavior: behavior ?? "followUp", ...withImages },
+        6e4
+      );
     } else {
-      await rpc.send({ type: "prompt", message }, 6e4);
+      await rpc.send({ type: "prompt", message: text, ...withImages }, 6e4);
     }
   }
   /**
@@ -1916,7 +1927,20 @@ ${rpc.stderrTail}` : ""}`;
         provider: String(model["provider"] ?? ""),
         id: String(model["id"] ?? ""),
         ...typeof model["name"] === "string" ? { name: model["name"] } : {},
-        ...typeof model["reasoning"] === "boolean" ? { reasoning: model["reasoning"] } : {}
+        ...typeof model["reasoning"] === "boolean" ? { reasoning: model["reasoning"] } : {},
+        /*
+         * `input` decides whether the composer may offer a paste.
+         *
+         * Filtered to the two values the wire declares rather than passed through: Pi's catalogue may grow a
+         * modality this version does not know, and an unrecognised member would widen the UI's type for a capability
+         * nothing here can use. Absent when Pi did not report it — which is "unknown", NOT "text only", and the
+         * difference decides whether the affordance is hidden or merely unflagged.
+         */
+        ...Array.isArray(model["input"]) ? {
+          input: model["input"].filter(
+            (value) => value === "text" || value === "image"
+          )
+        } : {}
       })).filter((model) => model.provider !== "" && model.id !== "");
       return { ok: true, models };
     } catch (error) {
@@ -2124,6 +2148,90 @@ function readPackageJson(path) {
 }
 var VERIFY_INSTALL_HINT = "npm audit signatures";
 
+// packages/orchestrator/src/prompt-images.ts
+var PROMPT_IMAGE_TYPES = [
+  "image/png",
+  "image/jpeg",
+  "image/gif",
+  "image/webp"
+];
+var MAX_PROMPT_IMAGE_BYTES = 3 * 1024 * 1024;
+function refuseImage(refusal, detail) {
+  switch (refusal) {
+    case "unsupported-type":
+      return `an image of type ${detail?.type ?? "unknown"} cannot be sent \u2014 prompts accept PNG, JPEG, GIF and WebP`;
+    case "too-large":
+      return `that image is ${formatMegabytes(detail?.bytes ?? 0)} decoded, above the ${formatMegabytes(MAX_PROMPT_IMAGE_BYTES)} limit for a prompt \u2014 downscale it and paste again`;
+    case "not-base64":
+      return "that image's data is not valid base64 \u2014 it may have been truncated in transit";
+    case "missing-type":
+      return "an image arrived without a media type";
+    case "missing-data":
+      return "an image arrived with no data";
+    case "not-an-object":
+      return "an image arrived in an unrecognised shape";
+  }
+}
+function formatMegabytes(bytes) {
+  if (!Number.isFinite(bytes) || bytes <= 0) return "an unknown size";
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+function base64Bytes(data) {
+  if (data.length === 0) return 0;
+  const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0;
+  return Math.floor(data.length * 3 / 4) - padding;
+}
+function isBase64(data) {
+  if (data.length === 0 || data.length % 4 !== 0) return false;
+  return /^[A-Za-z0-9+/]+={0,2}$/.test(data);
+}
+function acceptImage(raw) {
+  if (typeof raw !== "object" || raw === null) return { ok: false, reason: "not-an-object" };
+  const candidate = raw;
+  if (typeof candidate.mediaType !== "string" || candidate.mediaType.length === 0) {
+    return { ok: false, reason: "missing-type" };
+  }
+  if (typeof candidate.data !== "string" || candidate.data.length === 0) {
+    return { ok: false, reason: "missing-data" };
+  }
+  const mediaType = candidate.mediaType.toLowerCase();
+  if (!PROMPT_IMAGE_TYPES.includes(mediaType)) return { ok: false, reason: "unsupported-type" };
+  if (!isBase64(candidate.data)) return { ok: false, reason: "not-base64" };
+  const bytes = base64Bytes(candidate.data);
+  if (bytes > MAX_PROMPT_IMAGE_BYTES) return { ok: false, reason: "too-large" };
+  return { ok: true, image: { mediaType, data: candidate.data } };
+}
+var MAX_PROMPT_IMAGES_ENCODED_BYTES = 6 * 1024 * 1024;
+function acceptImages(raw) {
+  if (raw === void 0 || raw === null) return { ok: true, images: [] };
+  if (!Array.isArray(raw)) return { ok: false, message: "images must be a list" };
+  if (raw.length === 0) return { ok: true, images: [] };
+  const images = [];
+  for (const entry of raw) {
+    const result = acceptImage(entry);
+    if (!result.ok) {
+      const type = typeof entry === "object" && entry !== null ? entry.mediaType : void 0;
+      const bytes = typeof entry === "object" && entry !== null && typeof entry.data === "string" ? base64Bytes(entry.data) : void 0;
+      return {
+        ok: false,
+        message: refuseImage(result.reason, {
+          ...typeof type === "string" ? { type } : {},
+          ...bytes !== void 0 ? { bytes } : {}
+        })
+      };
+    }
+    images.push(result.image);
+  }
+  const encoded = images.reduce((total, image) => total + image.data.length, 0);
+  if (encoded > MAX_PROMPT_IMAGES_ENCODED_BYTES) {
+    return {
+      ok: false,
+      message: `these images total ${(encoded / (1024 * 1024)).toFixed(1)} MB encoded, above the ${(MAX_PROMPT_IMAGES_ENCODED_BYTES / (1024 * 1024)).toFixed(1)} MB limit for one prompt \u2014 remove one or downscale them`
+    };
+  }
+  return { ok: true, images };
+}
+
 // packages/orchestrator/src/registry.ts
 import { createReadStream, readdirSync as readdirSync3, realpathSync as realpathSync2, statSync as statSync6 } from "node:fs";
 import { basename, dirname as dirname2, join as join10, resolve as resolve7, sep as sep4 } from "node:path";
@@ -2751,13 +2859,15 @@ var Transport = class {
         this.#broadcast({ kind: "session", session: result.session });
         return;
       }
-      case "prompt":
-        await this.#supervisor.prompt(
-          control.sessionId,
-          control.message,
-          ...control.behavior ? [control.behavior] : []
-        );
+      case "prompt": {
+        const accepted = acceptImages(control.images);
+        if (!accepted.ok) {
+          this.#broadcast({ kind: "error", message: `image refused: ${accepted.message}`, sessionId: control.sessionId });
+          return;
+        }
+        await this.#supervisor.prompt(control.sessionId, control.message, control.behavior, accepted.images);
         return;
+      }
       case "clearQueue": {
         await this.#supervisor.clearQueue(control.sessionId, control.andStop);
         return;