privateer-agent 0.12.37 → 0.12.39

package/extensions/privateer-gate.ts

@@ -25,6 +25,7 @@ import { makeSaveCargoTool } from "../src/tools/cargo.ts";
 import { makeChartTools } from "../src/tools/charts.ts";
 import { makeSaveAttachmentTool } from "../src/tools/saveAttachment.ts";
 import { AttachmentStore, type StoredAttachment } from "../src/util/attachmentStore.ts";
+import { resolveMentions, searchFiles } from "../src/util/fileMentions.ts";
 import { makeExtensionsControl } from "../src/remote/extensionsControl.ts";
 import { makeSkillsControl } from "../src/remote/skillsControl.ts";
 import { agentDir } from "../src/config/paths.ts";
@@ -48,7 +49,9 @@ const allowedOutsideRoots: string[] = [];
 // SECOND prompt arriving while Pi is still processing — which throws "Agent is already
 // processing" and wedges the session. This happens in normal use when the app drops
 // (backgrounded → socket suspended) and re-sends its prompt on reconnect. Mirrors the
-// REPL's `turnActive` guard. Set on a successful sendUserMessage, cleared on agent_end.
+// REPL's `turnActive` guard. Claimed as soon as a prompt is accepted — ahead of the
+// mention expansion, which reads files and so opens a window a second prompt could
+// slip through — released again if the send fails, cleared on agent_end.
 let remoteTurnActive = false;
 
 let piRef: any = null;
@@ -149,7 +152,7 @@ async function switchModelRemote(spec: string): Promise<void> {
     const ok = await piRef?.setModel?.(model);
     if (ok === false) { relay?.sendNotice(`No API key for ${p} — can't switch to ${sp}.`); return; }
     currentSpec = sp;
-    relay?.sendContext({ model: currentSpec, version: agentVersion() }); // banner follows
+    relay?.sendContext({ model: currentSpec, cwd: process.cwd(), version: agentVersion() }); // banner follows
     relay?.sendNotice(`model → ${sp}`);
   } catch (e) {
     relay?.sendNotice(`Couldn't switch model: ${(e as Error).message}`);
@@ -344,6 +347,12 @@ const bridge = new RemoteBridge({
       relay?.sendNotice("busy — a turn is already running; wait for it to finish.");
       return;
     }
+    // Claim the turn BEFORE the awaits below, not after the send. Expanding mentions
+    // reads files, so the send is no longer synchronous with this callback — a second
+    // prompt arriving in that window would pass the guard above and land Pi with two
+    // turns. Released again on any failure path, so a refused send can't wedge the
+    // bridge; the success path leaves it set until agent_end.
+    remoteTurnActive = true;
     // Fold any files the app sent since the last prompt into a reference note so the
     // model knows they exist and can save_attachment them.
     const atts = sinceLastPrompt;
@@ -352,14 +361,32 @@ const bridge = new RemoteBridge({
       ? `\n\n[Files attached from the app: ${atts.map((a) => `#${a.n} ${a.name} (${a.mediaType})`).join(", ")}. ` +
         `Use the save_attachment tool with the ref number to write one to disk.]`
       : "";
-    try {
-      piRef?.sendUserMessage?.(text + note); // drive a turn in Pi's TUI
-      remoteTurnActive = true; // cleared on agent_end
-    } catch (e) {
-      // A synchronous "already processing" (or any send failure) must not wedge the
-      // bridge — surface it and stay idle so the next prompt still works.
-      relay?.sendNotice(`couldn't start turn: ${(e as Error).message}`);
-    }
+    void (async () => {
+      try {
+        // Expand any `@path` mentions into appended <file> blocks + image attachments,
+        // resolved against this terminal's cwd (constrained to the cwd subtree). The
+        // REPL (src/cli/chat.ts) and the desktop session (agentSession.ts) both do this;
+        // the shipped TUI did not, so the one surface with no Tab key — a phone driving
+        // this terminal — was the only one where `@file` did nothing at all.
+        const cwd = process.cwd();
+        const mentions = await resolveMentions(text, cwd);
+        if (mentions.skipped.length) {
+          relay?.sendNotice(`Couldn't attach: ${mentions.skipped.join(", ")} (must be a file inside ${cwd})`);
+        }
+        const body = mentions.text + note;
+        // Images ride as content parts — pi's sendUserMessage takes the same
+        // {type:"image",data,mimeType} shape resolveMentions already emits, so a
+        // mentioned screenshot reaches the model as a real attachment, not a path.
+        piRef?.sendUserMessage?.(
+          mentions.images.length ? [{ type: "text", text: body }, ...mentions.images] : body,
+        ); // drive a turn in Pi's TUI
+      } catch (e) {
+        // An "already processing" (or any send/expansion failure) must not wedge the
+        // bridge — surface it and stay idle so the next prompt still works.
+        remoteTurnActive = false;
+        relay?.sendNotice(`couldn't start turn: ${(e as Error).message}`);
+      }
+    })();
   },
   onInterrupt: () => {}, // Pi owns interrupt; best-effort no-op
   // The app asked to end remote access from its side — stop the relay locally too so
@@ -383,10 +410,23 @@ const bridge = new RemoteBridge({
     // terminal, and advertise the slash commands for the composer's autocomplete.
     setRemoteState("connected");
     relay?.sendSnapshot([{ kind: "notice", text: "Privateer terminal connected." }]);
-    relay?.sendContext({ model: currentSpec, version: agentVersion() });
+    // cwd rides along here (home-collapsed on the way out, see RelayClient.sendContext).
+    // Without it the app's composer shows no working-directory strip at all — and that
+    // strip is the only place a driver can see which folder the prompts they type are
+    // reading, writing and `@`-mentioning against.
+    relay?.sendContext({ model: currentSpec, cwd: process.cwd(), version: agentVersion() });
     relay?.sendCommands(advertiseCommands());
   },
   onAttachment: (file) => sinceLastPrompt.push(attachments.register(file)),
+  // The app composer is autocompleting an `@file` mention — list the cwd entries
+  // matching the query and reply. Read-only + cwd-constrained (searchFiles never
+  // escapes the subtree); resolution of the picked path happens in onPrompt above.
+  // Unanswered, the app's searchFiles() times out to [] after 4s and the palette
+  // reads "no files" — indistinguishable from an empty project.
+  onFilesSearch: (id, query) => void (async () => {
+    try { bridge.sendFileMatches(id, await searchFiles(query, process.cwd())); }
+    catch { bridge.sendFileMatches(id, []); }
+  })(),
   // Drive the indicator from the relay's own status stream: "connected" → green;
   // its reconnect/retry notices → yellow "connecting…". Ignored once we're off.
   onStatus: (text) => {
@@ -528,7 +568,7 @@ export default function privateerControl(pi: any): void {
   pi.on("model_select", (ev: any) => {
     if (ev?.model) {
       currentSpec = modelSpec(ev.model);
-      relay?.sendContext({ model: currentSpec, version: agentVersion() });
+      relay?.sendContext({ model: currentSpec, cwd: process.cwd(), version: agentVersion() });
     }
   });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "privateer-agent",
-  "version": "0.12.37",
+  "version": "0.12.39",
   "description": "Privacy-first terminal coding agent — bring your own model across 20 providers (Anthropic, OpenAI, OpenRouter, Google, local Ollama…). Safe-by-default permissions, MCP, sub-agents, workflows, and verifiable TEE inference. Built on the Pi toolkit.",
   "type": "module",
   "license": "MIT",

package/src/permissions/classify.ts

@@ -225,6 +225,9 @@ const MEDIA_TITLES: Record<string, string> = {
   video_compose: "Compose video/audio locally",
   media_capabilities: "Read media capabilities",
 };
+// Not in MEDIA_TITLES: it is the same tool, told apart by its arguments rather
+// than its name (see `resuming` below).
+const RESUME_VIDEO_TITLE = "Save a video already generated (nothing further is billed)";
 
 export function classifyToolCall(
   toolName: string,
@@ -478,6 +481,14 @@ export function classifyToolCall(
   // even when the output lands neatly in cwd.
   if (MEDIA_TOOLS.has(name)) {
     const compose = name === "video_compose";
+    // A generate_video RESUME submits nothing and bills nothing — it goes back to
+    // waiting on a job the account has already paid for and writes the file. So it
+    // is an ordinary write, not a billed one: the title must not claim a charge
+    // that isn't happening, and `alwaysAsk` must not make re-prompting the cheaper
+    // path than re-generating. Getting that backwards is what teaches a model to
+    // pay twice.
+    const resuming = name === "generate_video" && typeof obj.resumeJobId === "string" && !!obj.resumeJobId.trim();
+    const mediaTitle = resuming ? RESUME_VIDEO_TITLE : MEDIA_TITLES[name];
     const inputs = [
       ...(Array.isArray(obj.inputs) ? (obj.inputs as unknown[]).map(str) : []),
       str(obj.input),
@@ -549,13 +560,11 @@ export function classifyToolCall(
     return {
       tool: toolName,
       kind: "write",
-      title: outside
-        ? `${MEDIA_TITLES[name]} outside working directory`
-        : MEDIA_TITLES[name],
+      title: outside ? `${mediaTitle} outside working directory` : mediaTitle,
       detail: `${outputOutside ? absOut : outPath}${inputNote}`,
       protected: isProtectedPath(absOut) || protectedInputs.length > 0,
       outside,
-      alwaysAsk: BILLED_MEDIA_TOOLS.has(name),
+      alwaysAsk: BILLED_MEDIA_TOOLS.has(name) && !resuming,
       path: absOut,
     };
   }

package/src/tools/media.ts

@@ -56,8 +56,23 @@ export const MEDIA_TOOL_NAMES = [
 
 // A video job can legitimately take minutes. Bound the wait so a wedged provider
 // doesn't pin an unattended run forever; the job id is reported on timeout so the
-// caller can resume the poll rather than pay for another generation.
-const VIDEO_POLL_TIMEOUT_MS = Number(process.env.PRIVATEER_VIDEO_TIMEOUT_MS) || 12 * 60_000;
+// caller can resume the poll (`resumeJobId`) rather than pay for another generation.
+//
+// TWENTY-FIVE, AND THE RESUME PARAM, ARE ONE FIX. At twelve this was the only
+// surface that gave up on a job at all — the app's own poller (ChatScreen,
+// libraryService) runs on a bare setInterval with no deadline, so the identical
+// job that lands fine in a chat was abandoned in a terminal — and there was
+// nothing to resume WITH: `generate_video` took no job id, so the sentence above
+// described a recovery the tool could not perform. The model's only move was to
+// call generate_video again, which bills a second generation and starts a second
+// wait, which is what "video generation hangs" looks like from the outside.
+//
+// The ceiling stays because an unattended run must not be pinned forever, and 25
+// is not arbitrary: the desktop's turn supervisor abandons a turn whose open tool
+// has been silent for 30 minutes (desktop/src/main/turnSupervisor.ts toolStallMs),
+// and a tool that outlives its own supervisor is abandoned mid-poll with the job
+// uncollected — exactly the failure this is fixing. Keep it under that budget.
+const VIDEO_POLL_TIMEOUT_MS = Number(process.env.PRIVATEER_VIDEO_TIMEOUT_MS) || 25 * 60_000;
 const VIDEO_POLL_INTERVAL_MS = 5_000;
 // A mesh job runs about a minute at the provider's stated typical time, and
 // several for a large face count. Same bounded-wait contract as video: the job
@@ -334,10 +349,18 @@ export const generateVideoToolDefinition = {
     "extracted with video_compose, then start the next from it. Generation takes minutes and this tool " +
     "waits for it. Expensive (roughly $0.10-$1 a clip) and billed to the user's Privateer account, so " +
     "plan the shot before calling. Clip lengths and aspect ratios are model-specific — check " +
-    "media_capabilities first if unsure. Stitch the finished clips with video_compose.",
+    "media_capabilities first if unsure. Stitch the finished clips with video_compose.\n" +
+    "If a call comes back saying the job is still running, DO NOT call this again with the same " +
+    "prompt — that bills a second generation. Call it with `resumeJobId` set to the job id it " +
+    "reported (and the same `path`) to keep waiting on the clip the account has already paid for.",
   parameters: Type.Object({
     prompt: Type.String({ description: "What happens in the shot: subject, action, camera move, style." }),
     path: Type.String({ description: "Where to write the video, relative to cwd or absolute (e.g. 'clips/01-opening.mp4')." }),
+    resumeJobId: Type.Optional(Type.String({
+      description:
+        "Resume waiting on a job already submitted (from a previous call that timed out). " +
+        "Nothing is generated and nothing is billed: it only polls and saves. `prompt` is ignored.",
+    })),
     firstFrame: Type.Optional(Type.String({ description: "Path to an image to use as the opening frame (image-to-video)." })),
     lastFrame: Type.Optional(Type.String({ description: "Path to an image to use as the closing frame. Requires firstFrame." })),
     seconds: Type.Optional(Type.Number({ description: "Clip length in seconds. Only certain values are legal per model — see media_capabilities." })),
@@ -349,7 +372,7 @@ export const generateVideoToolDefinition = {
   async execute(
     _toolCallId: string,
     params: {
-      prompt: string; path: string; firstFrame?: string; lastFrame?: string;
+      prompt: string; path: string; resumeJobId?: string; firstFrame?: string; lastFrame?: string;
       seconds?: number; aspectRatio?: string; resolution?: string; audio?: boolean; model?: string;
     },
     signal?: AbortSignal,
@@ -357,9 +380,18 @@ export const generateVideoToolDefinition = {
     ctx?: { cwd?: string },
   ) {
     const cwd = ctx?.cwd ?? process.cwd();
+    if (!params.path) return text("Error: path is required — say where to save the video.");
+
+    // RESUME. Submit nothing, bill nothing — just go back to waiting on a job the
+    // account has already paid for. Everything below the poll is identical, which
+    // is why the loop lives in awaitVideoJob() rather than being duplicated here.
+    const resumeJobId = String(params.resumeJobId ?? "").trim();
+    if (resumeJobId) {
+      return awaitVideoJob(resumeJobId, cwd, params.path, null, signal);
+    }
+
     const prompt = String(params.prompt ?? "").trim();
     if (!prompt) return text("Error: prompt is required.");
-    if (!params.path) return text("Error: path is required — say where to save the video.");
     if (params.lastFrame && !params.firstFrame) return text("Error: lastFrame needs firstFrame alongside it.");
 
     let firstFrame: { data: string; mimeType: string } | undefined;
@@ -389,45 +421,72 @@ export const generateVideoToolDefinition = {
     const jobId = submitted.data.jobId;
     if (!jobId) return text("Video generation failed: Privateer did not return a job id.");
 
-    // Poll to completion. The account is charged when the provider delivers, so an
-    // abandoned poll still costs money — hence the timeout message names the job id.
-    const deadline = Date.now() + VIDEO_POLL_TIMEOUT_MS;
-    const cancelled = () =>
-      text(`Video job ${jobId} was submitted but the wait was cancelled. It is still running and will still be billed.`);
-    for (;;) {
-      if (signal?.aborted) return cancelled();
-      await sleep(VIDEO_POLL_INTERVAL_MS, signal);
-      // sleep() resolves early on abort, so re-check before spending a request on a
-      // signal that is already dead — otherwise the cancel surfaces as a network error.
-      if (signal?.aborted) return cancelled();
-      const poll = await callAccount<VideoStatusResponse>(`/api/agent/media/videos/${encodeURIComponent(jobId)}`, {
-        method: "GET",
-        signal,
-      });
-      if (!poll.ok) return text(`Video job ${jobId} could not be polled: ${poll.message}`);
+    return awaitVideoJob(jobId, cwd, params.path, submitted.data.model ?? null, signal);
+  },
+};
 
-      const status = String(poll.data.status ?? "").toLowerCase();
-      if (status === "failed") return text(`Video generation failed: ${poll.data.message ?? "the provider reported a failure"}.`);
-      if (status === "completed") {
-        if (!poll.data.data) {
-          // The bytes were handed out on an earlier poll and are not stored anywhere.
-          return text(`Video job ${jobId} already delivered its bytes on an earlier poll; they were not saved. Generate again if the file is missing.`);
-        }
-        const target = abs(cwd, params.path);
-        const ext = extname(target) || extForMime(poll.data.mimeType ?? "", ".mp4");
-        const out = `${target.slice(0, target.length - extname(target).length)}${ext}`;
-        const summary = writeOut(out, Buffer.from(poll.data.data, "base64"));
-        return text(`Generated video with ${poll.data.model ?? submitted.data.model ?? "the account video model"}: ${summary}`);
-      }
-      if (Date.now() > deadline) {
-        return text(
-          `Video job ${jobId} is still ${status || "running"} after ${Math.round(VIDEO_POLL_TIMEOUT_MS / 60000)} minutes. ` +
-            "It will still complete and still be billed; nothing was saved here.",
-        );
+/**
+ * Wait on a submitted video job and write its bytes to `path`.
+ *
+ * Split out of execute() so the RESUME path is the same code rather than a second
+ * copy of it: the bytes are delivered exactly once and are stored nowhere, so a
+ * resume that polled differently from the original wait would be the one place a
+ * paid clip could be dropped.
+ *
+ * `submittedModel` is null on a resume — we did not submit, so we have no model
+ * name of our own. The poll reports one anyway; the fallback only covers a server
+ * that returns neither.
+ */
+async function awaitVideoJob(
+  jobId: string,
+  cwd: string,
+  path: string,
+  submittedModel: string | null,
+  signal?: AbortSignal,
+) {
+  // The account is charged when the provider delivers, so an abandoned poll still
+  // costs money — every exit below names the job id, and `resumeJobId` is what
+  // turns that id back into the file.
+  const deadline = Date.now() + VIDEO_POLL_TIMEOUT_MS;
+  const resumeHint = `Resume it with generate_video { resumeJobId: "${jobId}", path: "${path}" } — that waits on this same clip and bills nothing further.`;
+  const cancelled = () =>
+    text(`Video job ${jobId} is still running and will still be billed; the wait was cancelled. ${resumeHint}`);
+  for (;;) {
+    if (signal?.aborted) return cancelled();
+    await sleep(VIDEO_POLL_INTERVAL_MS, signal);
+    // sleep() resolves early on abort, so re-check before spending a request on a
+    // signal that is already dead — otherwise the cancel surfaces as a network error.
+    if (signal?.aborted) return cancelled();
+    const poll = await callAccount<VideoStatusResponse>(`/api/agent/media/videos/${encodeURIComponent(jobId)}`, {
+      method: "GET",
+      signal,
+    });
+    // A poll that fails is NOT the job failing — the clip is still coming and is
+    // still billed, so this has to point at the resume too. Without that the model
+    // reads a dropped request as a dead job and generates the whole thing again.
+    if (!poll.ok) return text(`Video job ${jobId} could not be polled: ${poll.message}. ${resumeHint}`);
+
+    const status = String(poll.data.status ?? "").toLowerCase();
+    if (status === "failed") return text(`Video generation failed: ${poll.data.message ?? "the provider reported a failure"}.`);
+    if (status === "completed") {
+      if (!poll.data.data) {
+        // The bytes were handed out on an earlier poll and are not stored anywhere.
+        return text(`Video job ${jobId} already delivered its bytes on an earlier poll; they were not saved. Generate again if the file is missing.`);
       }
+      const target = abs(cwd, path);
+      const ext = extname(target) || extForMime(poll.data.mimeType ?? "", ".mp4");
+      const out = `${target.slice(0, target.length - extname(target).length)}${ext}`;
+      const summary = writeOut(out, Buffer.from(poll.data.data, "base64"));
+      return text(`Generated video with ${poll.data.model ?? submittedModel ?? "the account video model"}: ${summary}`);
     }
-  },
-};
+    if (Date.now() > deadline) {
+      return text(
+        `Video job ${jobId} is still ${status || "running"} after ${Math.round(VIDEO_POLL_TIMEOUT_MS / 60000)} minutes. ` +
+          `It will still complete and is already billed — do NOT generate it again. ${resumeHint}`,
+      );
+    }
+  }
+}
 
 function sleep(ms: number, signal?: AbortSignal): Promise<void> {
   return new Promise((resolve_) => {
@@ -1143,6 +1202,9 @@ export const generateSpriteToolDefinition = {
     "right-facing ones rather than rendered, which is why eight animations cost five clips and not " +
     "eight. Each clip is charged at the account's video rate, so an eight-way set is genuinely " +
     "expensive; say the total to the user before batching characters.\n" +
+    "TWO models are involved, which matters when one of them is down: an IMAGE model turns the " +
+    "picture to face each direction (only for `directions` 'four' and 'eight'), then a VIDEO model " +
+    "renders the motion per facing. `image_model` and `model` override them separately.\n" +
     "It takes several minutes (the clips render sequentially) and this tool waits. Frame count, cell " +
     "size and frame rate are chosen here and cost nothing extra. AVAILABILITY: this needs a video " +
     "decoder on the Privateer API and some deployments do not have one — call media_capabilities and " +
@@ -1218,13 +1280,25 @@ export const generateSpriteToolDefinition = {
     model: Type.Optional(
       Type.String({ description: "Video model id to render the motion with. Omit for the account default." }),
     ),
+    image_model: Type.Optional(
+      Type.String({
+        description:
+          "Image model id used to TURN the picture to face each direction, e.g. " +
+          "'google/gemini-3.1-flash-image'. Omit for the account default. Only used when " +
+          "`directions` is 'four' or 'eight' — 'one' renders no turns and never touches an image " +
+          "model. Worth setting when a run fails with SPRITE_IMAGE_MODEL_UNAVAILABLE: that is the " +
+          "image provider being down, not the request being wrong, and naming another model here " +
+          "retries the whole run for nothing (the turns are drawn before any clip is billed). " +
+          "media_capabilities lists the account's current image model.",
+      }),
+    ),
   }),
   async execute(
     _toolCallId: string,
     params: {
       image: string; prompt: string; dir: string; action?: string; directions?: string;
       frames?: number; frame_size?: number; fps?: number; loop?: boolean;
-      name?: string; res_path?: string; model?: string;
+      name?: string; res_path?: string; model?: string; image_model?: string;
     },
     signal?: AbortSignal,
     _onUpdate?: unknown,
@@ -1264,6 +1338,7 @@ export const generateSpriteToolDefinition = {
             return guessed ? { res_path: guessed } : {};
           })()),
         ...(params.model ? { model: params.model } : {}),
+        ...(params.image_model ? { image_model: params.image_model } : {}),
       },
     });
     if (!submitted.ok) return text(`Sprite generation failed: ${submitted.message}`);