privateer-agent 0.12.38 → 0.12.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/privateer-gate.ts +52 -12
- package/package.json +1 -1
- package/src/auth/accountSessions.ts +4 -3
- package/src/auth/privateer.ts +5 -5
- package/src/permissions/classify.ts +18 -4
- package/src/tools/media.ts +438 -142
|
@@ -25,6 +25,7 @@ import { makeSaveCargoTool } from "../src/tools/cargo.ts";
|
|
|
25
25
|
import { makeChartTools } from "../src/tools/charts.ts";
|
|
26
26
|
import { makeSaveAttachmentTool } from "../src/tools/saveAttachment.ts";
|
|
27
27
|
import { AttachmentStore, type StoredAttachment } from "../src/util/attachmentStore.ts";
|
|
28
|
+
import { resolveMentions, searchFiles } from "../src/util/fileMentions.ts";
|
|
28
29
|
import { makeExtensionsControl } from "../src/remote/extensionsControl.ts";
|
|
29
30
|
import { makeSkillsControl } from "../src/remote/skillsControl.ts";
|
|
30
31
|
import { agentDir } from "../src/config/paths.ts";
|
|
@@ -48,7 +49,9 @@ const allowedOutsideRoots: string[] = [];
|
|
|
48
49
|
// SECOND prompt arriving while Pi is still processing — which throws "Agent is already
|
|
49
50
|
// processing" and wedges the session. This happens in normal use when the app drops
|
|
50
51
|
// (backgrounded → socket suspended) and re-sends its prompt on reconnect. Mirrors the
|
|
51
|
-
// REPL's `turnActive` guard.
|
|
52
|
+
// REPL's `turnActive` guard. Claimed as soon as a prompt is accepted — ahead of the
|
|
53
|
+
// mention expansion, which reads files and so opens a window a second prompt could
|
|
54
|
+
// slip through — released again if the send fails, cleared on agent_end.
|
|
52
55
|
let remoteTurnActive = false;
|
|
53
56
|
|
|
54
57
|
let piRef: any = null;
|
|
@@ -149,7 +152,7 @@ async function switchModelRemote(spec: string): Promise<void> {
|
|
|
149
152
|
const ok = await piRef?.setModel?.(model);
|
|
150
153
|
if (ok === false) { relay?.sendNotice(`No API key for ${p} — can't switch to ${sp}.`); return; }
|
|
151
154
|
currentSpec = sp;
|
|
152
|
-
relay?.sendContext({ model: currentSpec, version: agentVersion() }); // banner follows
|
|
155
|
+
relay?.sendContext({ model: currentSpec, cwd: process.cwd(), version: agentVersion() }); // banner follows
|
|
153
156
|
relay?.sendNotice(`model → ${sp}`);
|
|
154
157
|
} catch (e) {
|
|
155
158
|
relay?.sendNotice(`Couldn't switch model: ${(e as Error).message}`);
|
|
@@ -344,6 +347,12 @@ const bridge = new RemoteBridge({
|
|
|
344
347
|
relay?.sendNotice("busy — a turn is already running; wait for it to finish.");
|
|
345
348
|
return;
|
|
346
349
|
}
|
|
350
|
+
// Claim the turn BEFORE the awaits below, not after the send. Expanding mentions
|
|
351
|
+
// reads files, so the send is no longer synchronous with this callback — a second
|
|
352
|
+
// prompt arriving in that window would pass the guard above and land Pi with two
|
|
353
|
+
// turns. Released again on any failure path, so a refused send can't wedge the
|
|
354
|
+
// bridge; the success path leaves it set until agent_end.
|
|
355
|
+
remoteTurnActive = true;
|
|
347
356
|
// Fold any files the app sent since the last prompt into a reference note so the
|
|
348
357
|
// model knows they exist and can save_attachment them.
|
|
349
358
|
const atts = sinceLastPrompt;
|
|
@@ -352,14 +361,32 @@ const bridge = new RemoteBridge({
|
|
|
352
361
|
? `\n\n[Files attached from the app: ${atts.map((a) => `#${a.n} ${a.name} (${a.mediaType})`).join(", ")}. ` +
|
|
353
362
|
`Use the save_attachment tool with the ref number to write one to disk.]`
|
|
354
363
|
: "";
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
364
|
+
void (async () => {
|
|
365
|
+
try {
|
|
366
|
+
// Expand any `@path` mentions into appended <file> blocks + image attachments,
|
|
367
|
+
// resolved against this terminal's cwd (constrained to the cwd subtree). The
|
|
368
|
+
// REPL (src/cli/chat.ts) and the desktop session (agentSession.ts) both do this;
|
|
369
|
+
// the shipped TUI did not, so the one surface with no Tab key — a phone driving
|
|
370
|
+
// this terminal — was the only one where `@file` did nothing at all.
|
|
371
|
+
const cwd = process.cwd();
|
|
372
|
+
const mentions = await resolveMentions(text, cwd);
|
|
373
|
+
if (mentions.skipped.length) {
|
|
374
|
+
relay?.sendNotice(`Couldn't attach: ${mentions.skipped.join(", ")} (must be a file inside ${cwd})`);
|
|
375
|
+
}
|
|
376
|
+
const body = mentions.text + note;
|
|
377
|
+
// Images ride as content parts — pi's sendUserMessage takes the same
|
|
378
|
+
// {type:"image",data,mimeType} shape resolveMentions already emits, so a
|
|
379
|
+
// mentioned screenshot reaches the model as a real attachment, not a path.
|
|
380
|
+
piRef?.sendUserMessage?.(
|
|
381
|
+
mentions.images.length ? [{ type: "text", text: body }, ...mentions.images] : body,
|
|
382
|
+
); // drive a turn in Pi's TUI
|
|
383
|
+
} catch (e) {
|
|
384
|
+
// An "already processing" (or any send/expansion failure) must not wedge the
|
|
385
|
+
// bridge — surface it and stay idle so the next prompt still works.
|
|
386
|
+
remoteTurnActive = false;
|
|
387
|
+
relay?.sendNotice(`couldn't start turn: ${(e as Error).message}`);
|
|
388
|
+
}
|
|
389
|
+
})();
|
|
363
390
|
},
|
|
364
391
|
onInterrupt: () => {}, // Pi owns interrupt; best-effort no-op
|
|
365
392
|
// The app asked to end remote access from its side — stop the relay locally too so
|
|
@@ -383,10 +410,23 @@ const bridge = new RemoteBridge({
|
|
|
383
410
|
// terminal, and advertise the slash commands for the composer's autocomplete.
|
|
384
411
|
setRemoteState("connected");
|
|
385
412
|
relay?.sendSnapshot([{ kind: "notice", text: "Privateer terminal connected." }]);
|
|
386
|
-
|
|
413
|
+
// cwd rides along here (home-collapsed on the way out, see RelayClient.sendContext).
|
|
414
|
+
// Without it the app's composer shows no working-directory strip at all — and that
|
|
415
|
+
// strip is the only place a driver can see which folder the prompts they type are
|
|
416
|
+
// reading, writing and `@`-mentioning against.
|
|
417
|
+
relay?.sendContext({ model: currentSpec, cwd: process.cwd(), version: agentVersion() });
|
|
387
418
|
relay?.sendCommands(advertiseCommands());
|
|
388
419
|
},
|
|
389
420
|
onAttachment: (file) => sinceLastPrompt.push(attachments.register(file)),
|
|
421
|
+
// The app composer is autocompleting an `@file` mention — list the cwd entries
|
|
422
|
+
// matching the query and reply. Read-only + cwd-constrained (searchFiles never
|
|
423
|
+
// escapes the subtree); resolution of the picked path happens in onPrompt above.
|
|
424
|
+
// Unanswered, the app's searchFiles() times out to [] after 4s and the palette
|
|
425
|
+
// reads "no files" — indistinguishable from an empty project.
|
|
426
|
+
onFilesSearch: (id, query) => void (async () => {
|
|
427
|
+
try { bridge.sendFileMatches(id, await searchFiles(query, process.cwd())); }
|
|
428
|
+
catch { bridge.sendFileMatches(id, []); }
|
|
429
|
+
})(),
|
|
390
430
|
// Drive the indicator from the relay's own status stream: "connected" → green;
|
|
391
431
|
// its reconnect/retry notices → yellow "connecting…". Ignored once we're off.
|
|
392
432
|
onStatus: (text) => {
|
|
@@ -528,7 +568,7 @@ export default function privateerControl(pi: any): void {
|
|
|
528
568
|
pi.on("model_select", (ev: any) => {
|
|
529
569
|
if (ev?.model) {
|
|
530
570
|
currentSpec = modelSpec(ev.model);
|
|
531
|
-
relay?.sendContext({ model: currentSpec, version: agentVersion() });
|
|
571
|
+
relay?.sendContext({ model: currentSpec, cwd: process.cwd(), version: agentVersion() });
|
|
532
572
|
}
|
|
533
573
|
});
|
|
534
574
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "privateer-agent",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.40",
|
|
4
4
|
"description": "Privacy-first terminal coding agent — bring your own model across 20 providers (Anthropic, OpenAI, OpenRouter, Google, local Ollama…). Safe-by-default permissions, MCP, sub-agents, workflows, and verifiable TEE inference. Built on the Pi toolkit.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
// only a CLEAN exit revoked it (session_shutdown → revokeLocalSessions). A terminal
|
|
6
6
|
// that dies without running its shutdown hook — SIGKILL, a closed window, a crash,
|
|
7
7
|
// `kill` — leaves its session row alive server-side for the rest of its ~24h TTL. Do
|
|
8
|
-
// that a few times and the
|
|
9
|
-
//
|
|
10
|
-
//
|
|
8
|
+
// that a few times and the account accumulates a pile of live rows nobody owns. (There
|
|
9
|
+
// used to be a per-device cap that refused the next spawn outright, which took the whole
|
|
10
|
+
// account channel down until the rows aged out; it is gone, and this is what keeps the
|
|
11
|
+
// row count sane without one.)
|
|
11
12
|
//
|
|
12
13
|
// The fix is to reclaim an orphan instead of stacking another row on top of it. That
|
|
13
14
|
// needs one bit the credential itself can't tell us: is the terminal that owns it
|
package/src/auth/privateer.ts
CHANGED
|
@@ -158,8 +158,9 @@ let _refreshInFlight: Promise<ChildSession> | null = null;
|
|
|
158
158
|
//
|
|
159
159
|
// That pairing only covers a CLEAN exit, though. A terminal killed without running its
|
|
160
160
|
// shutdown hook leaves its row alive server-side for the full TTL, and the next launch
|
|
161
|
-
// used to spawn another on top of it —
|
|
162
|
-
//
|
|
161
|
+
// used to spawn another on top of it — so a few crashes left a pile of live rows nobody
|
|
162
|
+
// owned (which used to be refused outright by a per-device cap the server has since
|
|
163
|
+
// dropped). So every session is also recorded in a pid-keyed registry
|
|
163
164
|
// (auth/accountSessions.ts) and acquireAccountCredential reclaims one whose owning
|
|
164
165
|
// terminal is gone instead of spawning. Keep the registry in step with reality:
|
|
165
166
|
// recordOwnedSession wherever a credential is minted or rotated, forgetOwnedSession
|
|
@@ -445,9 +446,8 @@ export async function runDeviceLogin(opts: {
|
|
|
445
446
|
//
|
|
446
447
|
// A 401 means the parent refresh token is gone — the machine login itself is dead, so
|
|
447
448
|
// clear it and announce (the UI flips to signed-out). EVERY OTHER status used to be
|
|
448
|
-
// reported as an expiry too, which actively misled:
|
|
449
|
-
//
|
|
450
|
-
// try again"), where /login is not the fix and the credentials are perfectly valid.
|
|
449
|
+
// reported as an expiry too, which actively misled: a rate limit or a server-side
|
|
450
|
+
// refusal is not something /login fixes, and the credentials are perfectly valid.
|
|
451
451
|
// Pass the server's own message through so the user learns what to actually do.
|
|
452
452
|
async function spawnFailure(res: Response): Promise<Error> {
|
|
453
453
|
if (res.status === 401) {
|
|
@@ -225,6 +225,10 @@ const MEDIA_TITLES: Record<string, string> = {
|
|
|
225
225
|
video_compose: "Compose video/audio locally",
|
|
226
226
|
media_capabilities: "Read media capabilities",
|
|
227
227
|
};
|
|
228
|
+
// Not in MEDIA_TITLES: they are the same tools, told apart by their arguments
|
|
229
|
+
// rather than their names (see `resuming` below).
|
|
230
|
+
const RESUME_VIDEO_TITLE = "Save a video already generated (nothing further is billed)";
|
|
231
|
+
const RESUME_SPRITE_TITLE = "Save a sprite animation already generated (nothing further is billed)";
|
|
228
232
|
|
|
229
233
|
export function classifyToolCall(
|
|
230
234
|
toolName: string,
|
|
@@ -478,6 +482,18 @@ export function classifyToolCall(
|
|
|
478
482
|
// even when the output lands neatly in cwd.
|
|
479
483
|
if (MEDIA_TOOLS.has(name)) {
|
|
480
484
|
const compose = name === "video_compose";
|
|
485
|
+
// A generate_video or generate_sprite RESUME submits nothing and bills nothing —
|
|
486
|
+
// it goes back to waiting on a job the account has already paid for and writes the
|
|
487
|
+
// files. So it is an ordinary write, not a billed one: the title must not claim a
|
|
488
|
+
// charge that isn't happening, and `alwaysAsk` must not make re-prompting the
|
|
489
|
+
// cheaper path than re-generating. Getting that backwards is what teaches a model
|
|
490
|
+
// to pay twice — and it costs the most on a sprite, where the alternative to a
|
|
491
|
+
// resume is a whole second fan-out of up to five video generations.
|
|
492
|
+
const resumable = name === "generate_video" || name === "generate_sprite";
|
|
493
|
+
const resuming = resumable && typeof obj.resumeJobId === "string" && !!obj.resumeJobId.trim();
|
|
494
|
+
const mediaTitle = resuming
|
|
495
|
+
? (name === "generate_sprite" ? RESUME_SPRITE_TITLE : RESUME_VIDEO_TITLE)
|
|
496
|
+
: MEDIA_TITLES[name];
|
|
481
497
|
const inputs = [
|
|
482
498
|
...(Array.isArray(obj.inputs) ? (obj.inputs as unknown[]).map(str) : []),
|
|
483
499
|
str(obj.input),
|
|
@@ -549,13 +565,11 @@ export function classifyToolCall(
|
|
|
549
565
|
return {
|
|
550
566
|
tool: toolName,
|
|
551
567
|
kind: "write",
|
|
552
|
-
title: outside
|
|
553
|
-
? `${MEDIA_TITLES[name]} outside working directory`
|
|
554
|
-
: MEDIA_TITLES[name],
|
|
568
|
+
title: outside ? `${mediaTitle} outside working directory` : mediaTitle,
|
|
555
569
|
detail: `${outputOutside ? absOut : outPath}${inputNote}`,
|
|
556
570
|
protected: isProtectedPath(absOut) || protectedInputs.length > 0,
|
|
557
571
|
outside,
|
|
558
|
-
alwaysAsk: BILLED_MEDIA_TOOLS.has(name),
|
|
572
|
+
alwaysAsk: BILLED_MEDIA_TOOLS.has(name) && !resuming,
|
|
559
573
|
path: absOut,
|
|
560
574
|
};
|
|
561
575
|
}
|
package/src/tools/media.ts
CHANGED
|
@@ -38,8 +38,9 @@
|
|
|
38
38
|
// stitch → score → send.
|
|
39
39
|
|
|
40
40
|
import { Type } from "typebox";
|
|
41
|
-
import { mkdirSync, readFileSync, writeFileSync, existsSync, statSync } from "node:fs";
|
|
42
|
-
import { dirname, extname, isAbsolute, resolve } from "node:path";
|
|
41
|
+
import { mkdirSync, readFileSync, writeFileSync, unlinkSync, existsSync, statSync } from "node:fs";
|
|
42
|
+
import { dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
43
|
+
import { tmpdir } from "node:os";
|
|
43
44
|
import { apiRequest } from "../auth/privateer.ts";
|
|
44
45
|
|
|
45
46
|
/** Tool names these definitions register, for allow-list construction. */
|
|
@@ -56,18 +57,37 @@ export const MEDIA_TOOL_NAMES = [
|
|
|
56
57
|
|
|
57
58
|
// A video job can legitimately take minutes. Bound the wait so a wedged provider
|
|
58
59
|
// doesn't pin an unattended run forever; the job id is reported on timeout so the
|
|
59
|
-
// caller can resume the poll rather than pay for another generation.
|
|
60
|
-
|
|
60
|
+
// caller can resume the poll (`resumeJobId`) rather than pay for another generation.
|
|
61
|
+
//
|
|
62
|
+
// TWENTY-FIVE, AND THE RESUME PARAM, ARE ONE FIX. At twelve this was the only
|
|
63
|
+
// surface that gave up on a job at all — the app's own poller (ChatScreen,
|
|
64
|
+
// libraryService) runs on a bare setInterval with no deadline, so the identical
|
|
65
|
+
// job that lands fine in a chat was abandoned in a terminal — and there was
|
|
66
|
+
// nothing to resume WITH: `generate_video` took no job id, so the sentence above
|
|
67
|
+
// described a recovery the tool could not perform. The model's only move was to
|
|
68
|
+
// call generate_video again, which bills a second generation and starts a second
|
|
69
|
+
// wait, which is what "video generation hangs" looks like from the outside.
|
|
70
|
+
//
|
|
71
|
+
// The ceiling stays because an unattended run must not be pinned forever, and 25
|
|
72
|
+
// is not arbitrary: the desktop's turn supervisor abandons a turn whose open tool
|
|
73
|
+
// has been silent for 30 minutes (desktop/src/main/turnSupervisor.ts toolStallMs),
|
|
74
|
+
// and a tool that outlives its own supervisor is abandoned mid-poll with the job
|
|
75
|
+
// uncollected — exactly the failure this is fixing. Keep it under that budget.
|
|
76
|
+
const VIDEO_POLL_TIMEOUT_MS = Number(process.env.PRIVATEER_VIDEO_TIMEOUT_MS) || 25 * 60_000;
|
|
61
77
|
const VIDEO_POLL_INTERVAL_MS = 5_000;
|
|
62
78
|
// A mesh job runs about a minute at the provider's stated typical time, and
|
|
63
79
|
// several for a large face count. Same bounded-wait contract as video: the job
|
|
64
80
|
// id is reported on timeout so the caller can resume the poll rather than pay
|
|
65
81
|
// for a second generation.
|
|
66
82
|
const MESH_POLL_TIMEOUT_MS = Number(process.env.PRIVATEER_MESH_TIMEOUT_MS) || 10 * 60_000;
|
|
67
|
-
// A sprite job renders one clip per BILLED facing,
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
// the
|
|
83
|
+
// A sprite job renders one clip per BILLED facing, so an eight-way set waits on
|
|
84
|
+
// five video generations rather than one. They are SUBMITTED in one pass and
|
|
85
|
+
// render together at the provider (spriteApiHandler submits the whole fan-out
|
|
86
|
+
// before returning the 202), so the wait is roughly one clip's — what is serial
|
|
87
|
+
// is the stage BEFORE it, where the picture is turned to face each direction one
|
|
88
|
+
// edit at a time. The ceiling is correspondingly generous; as with video, the job
|
|
89
|
+
// id is reported on timeout and `resumeJobId` is what goes back to it, so a slow
|
|
90
|
+
// job is never a reason to pay for a second fan-out.
|
|
71
91
|
const SPRITE_POLL_TIMEOUT_MS = Number(process.env.PRIVATEER_SPRITE_TIMEOUT_MS) || 25 * 60_000;
|
|
72
92
|
const SPRITE_POLL_INTERVAL_MS = 6_000;
|
|
73
93
|
const MESH_POLL_INTERVAL_MS = 5_000;
|
|
@@ -135,6 +155,15 @@ function readInputImage(cwd: string, p: string): { data: string; mimeType: strin
|
|
|
135
155
|
interface AccountFailure {
|
|
136
156
|
ok: false;
|
|
137
157
|
message: string;
|
|
158
|
+
/**
|
|
159
|
+
* The HTTP status behind the failure, or undefined when the request never got
|
|
160
|
+
* an answer at all (DNS, reset, offline). A POLLER needs this and a one-shot
|
|
161
|
+
* caller does not: abandoning a paid job because one poll returned 502 throws
|
|
162
|
+
* the job away, while retrying a 404 or a 410 forever is just as wrong. The
|
|
163
|
+
* status is the only thing that separates the two, so it is carried rather
|
|
164
|
+
* than flattened into the message.
|
|
165
|
+
*/
|
|
166
|
+
status?: number;
|
|
138
167
|
}
|
|
139
168
|
|
|
140
169
|
/**
|
|
@@ -182,31 +211,31 @@ async function callAccount<T>(
|
|
|
182
211
|
// by the user (not by the model) — say exactly which switch to change and stop.
|
|
183
212
|
if (code === "ZDR_MEDIA_BLOCKED") {
|
|
184
213
|
return {
|
|
185
|
-
ok: false,
|
|
214
|
+
ok: false, status: res.status,
|
|
186
215
|
message:
|
|
187
216
|
(serverMessage || "this account requires Zero Data Retention and the media model has no ZDR endpoint") +
|
|
188
217
|
" — this is a privacy setting only the account owner can change (Settings → Privacy), so do not retry.",
|
|
189
218
|
};
|
|
190
219
|
}
|
|
191
220
|
if (code === "ZDR_KEY_UNAVAILABLE") {
|
|
192
|
-
return { ok: false, message: serverMessage || "no zero-retention provider key is available right now — try again later" };
|
|
221
|
+
return { ok: false, status: res.status, message: serverMessage || "no zero-retention provider key is available right now — try again later" };
|
|
193
222
|
}
|
|
194
223
|
if (/DAILY_CAP|LIMIT_REACHED/i.test(code) || res.status === 429) {
|
|
195
|
-
return { ok: false, message: serverMessage || "the account's daily media allowance is used up — it resets tomorrow" };
|
|
224
|
+
return { ok: false, status: res.status, message: serverMessage || "the account's daily media allowance is used up — it resets tomorrow" };
|
|
196
225
|
}
|
|
197
226
|
if (res.status === 402 || /INSUFFICIENT|QUOTA|TOP_?UP/i.test(code)) {
|
|
198
|
-
return { ok: false, message: serverMessage || "the account is out of credit for media generation — top up or upgrade to continue" };
|
|
227
|
+
return { ok: false, status: res.status, message: serverMessage || "the account is out of credit for media generation — top up or upgrade to continue" };
|
|
199
228
|
}
|
|
200
229
|
if (res.status === 401 || res.status === 403) {
|
|
201
230
|
return {
|
|
202
|
-
ok: false,
|
|
231
|
+
ok: false, status: res.status,
|
|
203
232
|
message:
|
|
204
233
|
serverMessage ||
|
|
205
234
|
"this agent is not signed in to a Privateer account (or the plan doesn't include this), so it cannot generate media",
|
|
206
235
|
};
|
|
207
236
|
}
|
|
208
237
|
if (res.status === 400 || res.status === 413) {
|
|
209
|
-
return { ok: false, message: serverMessage || `Privateer rejected the request${code ? ` (${code})` : ""}` };
|
|
238
|
+
return { ok: false, status: res.status, message: serverMessage || `Privateer rejected the request${code ? ` (${code})` : ""}` };
|
|
210
239
|
}
|
|
211
240
|
// 503/504 are OUR outage or a provider timing out, not a bad request: an unset
|
|
212
241
|
// provider key, our own balance with that provider, or a slow job. Retrying the same
|
|
@@ -214,11 +243,11 @@ async function callAccount<T>(
|
|
|
214
243
|
// good prompt in the belief it caused this.
|
|
215
244
|
if (res.status === 503 || res.status === 504) {
|
|
216
245
|
return {
|
|
217
|
-
ok: false,
|
|
246
|
+
ok: false, status: res.status,
|
|
218
247
|
message: `${serverMessage || "that media service is temporarily unavailable"} — this is on Privateer's side, not the prompt's; try again in a few minutes`,
|
|
219
248
|
};
|
|
220
249
|
}
|
|
221
|
-
return { ok: false, message: serverMessage || `media generation failed (HTTP ${res.status}${code ? ` ${code}` : ""})` };
|
|
250
|
+
return { ok: false, status: res.status, message: serverMessage || `media generation failed (HTTP ${res.status}${code ? ` ${code}` : ""})` };
|
|
222
251
|
}
|
|
223
252
|
|
|
224
253
|
// ── Images ───────────────────────────────────────────────────────────────────
|
|
@@ -334,10 +363,18 @@ export const generateVideoToolDefinition = {
|
|
|
334
363
|
"extracted with video_compose, then start the next from it. Generation takes minutes and this tool " +
|
|
335
364
|
"waits for it. Expensive (roughly $0.10-$1 a clip) and billed to the user's Privateer account, so " +
|
|
336
365
|
"plan the shot before calling. Clip lengths and aspect ratios are model-specific — check " +
|
|
337
|
-
"media_capabilities first if unsure. Stitch the finished clips with video_compose
|
|
366
|
+
"media_capabilities first if unsure. Stitch the finished clips with video_compose.\n" +
|
|
367
|
+
"If a call comes back saying the job is still running, DO NOT call this again with the same " +
|
|
368
|
+
"prompt — that bills a second generation. Call it with `resumeJobId` set to the job id it " +
|
|
369
|
+
"reported (and the same `path`) to keep waiting on the clip the account has already paid for.",
|
|
338
370
|
parameters: Type.Object({
|
|
339
371
|
prompt: Type.String({ description: "What happens in the shot: subject, action, camera move, style." }),
|
|
340
372
|
path: Type.String({ description: "Where to write the video, relative to cwd or absolute (e.g. 'clips/01-opening.mp4')." }),
|
|
373
|
+
resumeJobId: Type.Optional(Type.String({
|
|
374
|
+
description:
|
|
375
|
+
"Resume waiting on a job already submitted (from a previous call that timed out). " +
|
|
376
|
+
"Nothing is generated and nothing is billed: it only polls and saves. `prompt` is ignored.",
|
|
377
|
+
})),
|
|
341
378
|
firstFrame: Type.Optional(Type.String({ description: "Path to an image to use as the opening frame (image-to-video)." })),
|
|
342
379
|
lastFrame: Type.Optional(Type.String({ description: "Path to an image to use as the closing frame. Requires firstFrame." })),
|
|
343
380
|
seconds: Type.Optional(Type.Number({ description: "Clip length in seconds. Only certain values are legal per model — see media_capabilities." })),
|
|
@@ -349,7 +386,7 @@ export const generateVideoToolDefinition = {
|
|
|
349
386
|
async execute(
|
|
350
387
|
_toolCallId: string,
|
|
351
388
|
params: {
|
|
352
|
-
prompt: string; path: string; firstFrame?: string; lastFrame?: string;
|
|
389
|
+
prompt: string; path: string; resumeJobId?: string; firstFrame?: string; lastFrame?: string;
|
|
353
390
|
seconds?: number; aspectRatio?: string; resolution?: string; audio?: boolean; model?: string;
|
|
354
391
|
},
|
|
355
392
|
signal?: AbortSignal,
|
|
@@ -357,9 +394,18 @@ export const generateVideoToolDefinition = {
|
|
|
357
394
|
ctx?: { cwd?: string },
|
|
358
395
|
) {
|
|
359
396
|
const cwd = ctx?.cwd ?? process.cwd();
|
|
397
|
+
if (!params.path) return text("Error: path is required — say where to save the video.");
|
|
398
|
+
|
|
399
|
+
// RESUME. Submit nothing, bill nothing — just go back to waiting on a job the
|
|
400
|
+
// account has already paid for. Everything below the poll is identical, which
|
|
401
|
+
// is why the loop lives in awaitVideoJob() rather than being duplicated here.
|
|
402
|
+
const resumeJobId = String(params.resumeJobId ?? "").trim();
|
|
403
|
+
if (resumeJobId) {
|
|
404
|
+
return awaitVideoJob(resumeJobId, cwd, params.path, null, signal);
|
|
405
|
+
}
|
|
406
|
+
|
|
360
407
|
const prompt = String(params.prompt ?? "").trim();
|
|
361
408
|
if (!prompt) return text("Error: prompt is required.");
|
|
362
|
-
if (!params.path) return text("Error: path is required — say where to save the video.");
|
|
363
409
|
if (params.lastFrame && !params.firstFrame) return text("Error: lastFrame needs firstFrame alongside it.");
|
|
364
410
|
|
|
365
411
|
let firstFrame: { data: string; mimeType: string } | undefined;
|
|
@@ -389,45 +435,72 @@ export const generateVideoToolDefinition = {
|
|
|
389
435
|
const jobId = submitted.data.jobId;
|
|
390
436
|
if (!jobId) return text("Video generation failed: Privateer did not return a job id.");
|
|
391
437
|
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
const cancelled = () =>
|
|
396
|
-
text(`Video job ${jobId} was submitted but the wait was cancelled. It is still running and will still be billed.`);
|
|
397
|
-
for (;;) {
|
|
398
|
-
if (signal?.aborted) return cancelled();
|
|
399
|
-
await sleep(VIDEO_POLL_INTERVAL_MS, signal);
|
|
400
|
-
// sleep() resolves early on abort, so re-check before spending a request on a
|
|
401
|
-
// signal that is already dead — otherwise the cancel surfaces as a network error.
|
|
402
|
-
if (signal?.aborted) return cancelled();
|
|
403
|
-
const poll = await callAccount<VideoStatusResponse>(`/api/agent/media/videos/${encodeURIComponent(jobId)}`, {
|
|
404
|
-
method: "GET",
|
|
405
|
-
signal,
|
|
406
|
-
});
|
|
407
|
-
if (!poll.ok) return text(`Video job ${jobId} could not be polled: ${poll.message}`);
|
|
438
|
+
return awaitVideoJob(jobId, cwd, params.path, submitted.data.model ?? null, signal);
|
|
439
|
+
},
|
|
440
|
+
};
|
|
408
441
|
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
442
|
+
/**
|
|
443
|
+
* Wait on a submitted video job and write its bytes to `path`.
|
|
444
|
+
*
|
|
445
|
+
* Split out of execute() so the RESUME path is the same code rather than a second
|
|
446
|
+
* copy of it: the bytes are delivered exactly once and are stored nowhere, so a
|
|
447
|
+
* resume that polled differently from the original wait would be the one place a
|
|
448
|
+
* paid clip could be dropped.
|
|
449
|
+
*
|
|
450
|
+
* `submittedModel` is null on a resume — we did not submit, so we have no model
|
|
451
|
+
* name of our own. The poll reports one anyway; the fallback only covers a server
|
|
452
|
+
* that returns neither.
|
|
453
|
+
*/
|
|
454
|
+
async function awaitVideoJob(
|
|
455
|
+
jobId: string,
|
|
456
|
+
cwd: string,
|
|
457
|
+
path: string,
|
|
458
|
+
submittedModel: string | null,
|
|
459
|
+
signal?: AbortSignal,
|
|
460
|
+
) {
|
|
461
|
+
// The account is charged when the provider delivers, so an abandoned poll still
|
|
462
|
+
// costs money — every exit below names the job id, and `resumeJobId` is what
|
|
463
|
+
// turns that id back into the file.
|
|
464
|
+
const deadline = Date.now() + VIDEO_POLL_TIMEOUT_MS;
|
|
465
|
+
const resumeHint = `Resume it with generate_video { resumeJobId: "${jobId}", path: "${path}" } — that waits on this same clip and bills nothing further.`;
|
|
466
|
+
const cancelled = () =>
|
|
467
|
+
text(`Video job ${jobId} is still running and will still be billed; the wait was cancelled. ${resumeHint}`);
|
|
468
|
+
for (;;) {
|
|
469
|
+
if (signal?.aborted) return cancelled();
|
|
470
|
+
await sleep(VIDEO_POLL_INTERVAL_MS, signal);
|
|
471
|
+
// sleep() resolves early on abort, so re-check before spending a request on a
|
|
472
|
+
// signal that is already dead — otherwise the cancel surfaces as a network error.
|
|
473
|
+
if (signal?.aborted) return cancelled();
|
|
474
|
+
const poll = await callAccount<VideoStatusResponse>(`/api/agent/media/videos/${encodeURIComponent(jobId)}`, {
|
|
475
|
+
method: "GET",
|
|
476
|
+
signal,
|
|
477
|
+
});
|
|
478
|
+
// A poll that fails is NOT the job failing — the clip is still coming and is
|
|
479
|
+
// still billed, so this has to point at the resume too. Without that the model
|
|
480
|
+
// reads a dropped request as a dead job and generates the whole thing again.
|
|
481
|
+
if (!poll.ok) return text(`Video job ${jobId} could not be polled: ${poll.message}. ${resumeHint}`);
|
|
482
|
+
|
|
483
|
+
const status = String(poll.data.status ?? "").toLowerCase();
|
|
484
|
+
if (status === "failed") return text(`Video generation failed: ${poll.data.message ?? "the provider reported a failure"}.`);
|
|
485
|
+
if (status === "completed") {
|
|
486
|
+
if (!poll.data.data) {
|
|
487
|
+
// The bytes were handed out on an earlier poll and are not stored anywhere.
|
|
488
|
+
return text(`Video job ${jobId} already delivered its bytes on an earlier poll; they were not saved. Generate again if the file is missing.`);
|
|
427
489
|
}
|
|
490
|
+
const target = abs(cwd, path);
|
|
491
|
+
const ext = extname(target) || extForMime(poll.data.mimeType ?? "", ".mp4");
|
|
492
|
+
const out = `${target.slice(0, target.length - extname(target).length)}${ext}`;
|
|
493
|
+
const summary = writeOut(out, Buffer.from(poll.data.data, "base64"));
|
|
494
|
+
return text(`Generated video with ${poll.data.model ?? submittedModel ?? "the account video model"}: ${summary}`);
|
|
428
495
|
}
|
|
429
|
-
|
|
430
|
-
|
|
496
|
+
if (Date.now() > deadline) {
|
|
497
|
+
return text(
|
|
498
|
+
`Video job ${jobId} is still ${status || "running"} after ${Math.round(VIDEO_POLL_TIMEOUT_MS / 60000)} minutes. ` +
|
|
499
|
+
`It will still complete and is already billed — do NOT generate it again. ${resumeHint}`,
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
}
|
|
431
504
|
|
|
432
505
|
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
433
506
|
return new Promise((resolve_) => {
|
|
@@ -1065,6 +1138,44 @@ interface SpriteStatusResponse {
|
|
|
1065
1138
|
message?: string;
|
|
1066
1139
|
}
|
|
1067
1140
|
|
|
1141
|
+
/**
|
|
1142
|
+
* Is `target` the directory `root` itself, or something inside it?
|
|
1143
|
+
*
|
|
1144
|
+
* Asked through `relative` rather than by comparing string prefixes, because
|
|
1145
|
+
* the obvious `target.startsWith(root + "/")` is WRONG for the one root whose
|
|
1146
|
+
* own spelling already ends in a separator: with root `/`, every entry in a
|
|
1147
|
+
* perfectly ordinary archive resolves to `/thing` and matches no prefix `//`,
|
|
1148
|
+
* so a bundle destined for the filesystem root was rejected entry-by-entry as
|
|
1149
|
+
* an escape attempt. That is not a hypothetical — an agent whose cwd is `/`
|
|
1150
|
+
* (an Electron app opened from the Finder, a daemon started by launchd) and a
|
|
1151
|
+
* `dir` of `.` lands exactly there, and the sprite bundle it had already paid
|
|
1152
|
+
* for was thrown away with a security error that named the wrong problem. The
|
|
1153
|
+
* prefix form is also separator-blind on Windows.
|
|
1154
|
+
*/
|
|
1155
|
+
export function pathInside(root: string, target: string): boolean {
|
|
1156
|
+
const rel = relative(root, target);
|
|
1157
|
+
return rel === "" || (!isAbsolute(rel) && rel !== ".." && !rel.startsWith(".." + sep));
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
/**
|
|
1161
|
+
* Turn one archive entry name into a path under `root`, or refuse it.
|
|
1162
|
+
*
|
|
1163
|
+
* Separators are normalised first so a `..\\..` written the Windows way is
|
|
1164
|
+
* judged as the traversal it is on every platform. The refusal names the
|
|
1165
|
+
* destination as well as the entry, because the entry name alone is the half
|
|
1166
|
+
* the reader already has: it is WHERE the archive was being unpacked to that
|
|
1167
|
+
* says whether the bundle or the caller's `dir` is the thing at fault.
|
|
1168
|
+
*/
|
|
1169
|
+
function archiveEntryPath(name: string, root: string): string {
|
|
1170
|
+
const rel = name.split("\\").join("/");
|
|
1171
|
+
const traverses =
|
|
1172
|
+
rel.startsWith("/") || /^[A-Za-z]:/.test(rel) || rel.split("/").some((seg) => seg === "..");
|
|
1173
|
+
if (traverses || !pathInside(root, resolve(root, rel))) {
|
|
1174
|
+
throw new Error(`archive entry "${name}" escapes the destination directory ${root}`);
|
|
1175
|
+
}
|
|
1176
|
+
return rel;
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1068
1179
|
/**
|
|
1069
1180
|
* Unpack the bundle into a directory.
|
|
1070
1181
|
*
|
|
@@ -1073,10 +1184,10 @@ interface SpriteStatusResponse {
|
|
|
1073
1184
|
* payload is already PNG, so deflating it twice buys nothing), which means every
|
|
1074
1185
|
* entry is a header followed by its bytes verbatim.
|
|
1075
1186
|
*
|
|
1076
|
-
* ZIP-SLIP: entry names come off the wire, so each
|
|
1077
|
-
*
|
|
1078
|
-
* would let a generated archive write anywhere the agent can
|
|
1079
|
-
* unattended run is the user's whole machine.
|
|
1187
|
+
* ZIP-SLIP: entry names come off the wire, so each one is checked to be a plain
|
|
1188
|
+
* relative path landing inside the destination before anything is written. A
|
|
1189
|
+
* `..` segment here would let a generated archive write anywhere the agent can
|
|
1190
|
+
* reach, which on an unattended run is the user's whole machine.
|
|
1080
1191
|
*/
|
|
1081
1192
|
export function extractStoredZip(zip: Buffer, destDir: string): string[] {
|
|
1082
1193
|
const written: string[] = [];
|
|
@@ -1094,10 +1205,7 @@ export function extractStoredZip(zip: Buffer, destDir: string): string[] {
|
|
|
1094
1205
|
if (method !== 0) throw new Error(`archive entry "${name}" is compressed; only stored entries are expected`);
|
|
1095
1206
|
if (dataAt + size > zip.length) throw new Error(`archive entry "${name}" is truncated`);
|
|
1096
1207
|
|
|
1097
|
-
const target = resolve(root, name);
|
|
1098
|
-
if (target !== root && !target.startsWith(root + "/")) {
|
|
1099
|
-
throw new Error(`archive entry "${name}" escapes the destination directory`);
|
|
1100
|
-
}
|
|
1208
|
+
const target = resolve(root, archiveEntryPath(name, root));
|
|
1101
1209
|
mkdirSync(dirname(target), { recursive: true });
|
|
1102
1210
|
writeFileSync(target, zip.subarray(dataAt, dataAt + size));
|
|
1103
1211
|
written.push(name);
|
|
@@ -1122,8 +1230,205 @@ export function extractStoredZip(zip: Buffer, destDir: string): string[] {
|
|
|
1122
1230
|
export function guessResPath(cwd: string, dir: string): string | undefined {
|
|
1123
1231
|
const target = resolve(abs(cwd, dir));
|
|
1124
1232
|
const root = resolve(cwd);
|
|
1125
|
-
if (target === root || !
|
|
1126
|
-
return `res://${
|
|
1233
|
+
if (target === root || !pathInside(root, target)) return undefined;
|
|
1234
|
+
return `res://${relative(root, target).split("\\").join("/")}/`;
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
/**
|
|
1238
|
+
* The direction sets the server actually recognises, with what each one yields
|
|
1239
|
+
* and what it bills.
|
|
1240
|
+
*
|
|
1241
|
+
* Kept here as a table rather than trusted to the description, because the
|
|
1242
|
+
* server does not reject an unknown set — `parseSpec` falls back to 'one'. So
|
|
1243
|
+
* `directions: "4"`, `"four-way"` or `"down,left,right,up"` used to be accepted
|
|
1244
|
+
* all the way through, bill one clip, and hand back a single-facing sheet with
|
|
1245
|
+
* no error anywhere: the caller asked for a four-way walk cycle and got one
|
|
1246
|
+
* animation, which reads as the feature being broken rather than the argument
|
|
1247
|
+
* being wrong. The schema is a closed set now and this table is what the
|
|
1248
|
+
* unpacked result is checked against.
|
|
1249
|
+
*/
|
|
1250
|
+
const SPRITE_DIRECTION_SETS = {
|
|
1251
|
+
one: { animations: 1, billed: 1 },
|
|
1252
|
+
four: { animations: 4, billed: 3 },
|
|
1253
|
+
eight: { animations: 8, billed: 5 },
|
|
1254
|
+
} as const;
|
|
1255
|
+
type SpriteDirections = keyof typeof SPRITE_DIRECTION_SETS;
|
|
1256
|
+
|
|
1257
|
+
/**
|
|
1258
|
+
* Whether a failed poll is worth another try.
|
|
1259
|
+
*
|
|
1260
|
+
* A poll that fails is NOT the job failing — the clips are still rendering at
|
|
1261
|
+
* the provider and are still being paid for. A transport error (no status at
|
|
1262
|
+
* all), a 429 or any 5xx is our side or the network having a bad moment, and the
|
|
1263
|
+
* next poll six seconds later will very likely work; abandoning the job on the
|
|
1264
|
+
* first one throws away up to five video generations the account has already
|
|
1265
|
+
* been charged for, and on this path the bundle is delivered ONCE and stored
|
|
1266
|
+
* nowhere, so there is nothing to go back for. A 404/410/401/402 is the
|
|
1267
|
+
* opposite: the job is gone, already delivered, or was never ours, and waiting
|
|
1268
|
+
* changes nothing.
|
|
1269
|
+
*/
|
|
1270
|
+
export function spritePollWorthRetrying(status?: number): boolean {
|
|
1271
|
+
return status === undefined || status === 429 || status >= 500;
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
// How many consecutive failed polls to ride out before giving up. Six seconds
|
|
1275
|
+
// apart, so this is a minute of Privateer being unreachable — long enough to
|
|
1276
|
+
// cover a deploy or a blip, short enough that a genuinely dead endpoint does not
|
|
1277
|
+
// hold an unattended run until the 25-minute ceiling.
|
|
1278
|
+
const SPRITE_POLL_FAILURE_BUDGET = 10;
|
|
1279
|
+
|
|
1280
|
+
/**
|
|
1281
|
+
* Put the delivered archive somewhere safe before unpacking it.
|
|
1282
|
+
*
|
|
1283
|
+
* Returns the path, or null if even this failed — in which case the caller is
|
|
1284
|
+
* out of options and has to say so. Deliberately the temp directory rather than
|
|
1285
|
+
* the destination: the destination is the thing that may be unwritable or wrong,
|
|
1286
|
+
* and a stray .zip inside a Godot project gets picked up by the import scan.
|
|
1287
|
+
*/
|
|
1288
|
+
function stashBundle(bundle: Buffer, jobId: string): string | null {
|
|
1289
|
+
try {
|
|
1290
|
+
const path = join(tmpdir(), `privateer-sprite-${jobId.replace(/[^A-Za-z0-9_-]/g, "")}.zip`);
|
|
1291
|
+
writeFileSync(path, bundle);
|
|
1292
|
+
return path;
|
|
1293
|
+
} catch {
|
|
1294
|
+
return null;
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1298
|
+
/**
|
|
1299
|
+
* Wait on a submitted sprite job, unpack the bundle, and describe what landed.
|
|
1300
|
+
*
|
|
1301
|
+
* Split out of `execute` for the reason `awaitVideoJob` is: `resumeJobId` has to
|
|
1302
|
+
* poll EXACTLY as the original call did, and a resume that polled differently
|
|
1303
|
+
* would be the one path nobody exercises until someone's five-clip job is on the
|
|
1304
|
+
* line.
|
|
1305
|
+
*
|
|
1306
|
+
* `expectedAnimations` is the size of the direction set that was asked for, and
|
|
1307
|
+
* is null on a resume (where the request that chose it is gone). When it is
|
|
1308
|
+
* known it is checked against what actually came back — the server packs a sheet
|
|
1309
|
+
* out of whatever facings rendered and silently drops the rest, so a four-way
|
|
1310
|
+
* set whose "up" clip failed returns three animations, bills three, and says
|
|
1311
|
+
* nothing. That sheet is not the one the caller asked for and the .tres does not
|
|
1312
|
+
* contain the animation their GDScript will play.
|
|
1313
|
+
*/
|
|
1314
|
+
async function awaitSpriteJob(
|
|
1315
|
+
jobId: string,
|
|
1316
|
+
cwd: string,
|
|
1317
|
+
dir: string,
|
|
1318
|
+
expectedAnimations: number | null,
|
|
1319
|
+
billed: number | null,
|
|
1320
|
+
signal?: AbortSignal,
|
|
1321
|
+
) {
|
|
1322
|
+
const deadline = Date.now() + SPRITE_POLL_TIMEOUT_MS;
|
|
1323
|
+
// The clips are charged as they land, so an abandoned poll still costs money —
|
|
1324
|
+
// hence every exit below names the job id and says how to get back to it
|
|
1325
|
+
// without paying twice.
|
|
1326
|
+
const resumeHint =
|
|
1327
|
+
`Resume it with generate_sprite { resumeJobId: "${jobId}", dir: "${dir}" } — ` +
|
|
1328
|
+
"that waits on this same job and bills nothing further.";
|
|
1329
|
+
const cancelled = () =>
|
|
1330
|
+
text(
|
|
1331
|
+
`Sprite job ${jobId} was submitted but the wait was cancelled. Its ${billed ?? "queued"} clip(s) are still rendering and will still be billed. ` +
|
|
1332
|
+
resumeHint,
|
|
1333
|
+
);
|
|
1334
|
+
|
|
1335
|
+
let consecutiveFailures = 0;
|
|
1336
|
+
|
|
1337
|
+
for (;;) {
|
|
1338
|
+
if (signal?.aborted) return cancelled();
|
|
1339
|
+
await sleep(SPRITE_POLL_INTERVAL_MS, signal);
|
|
1340
|
+
if (signal?.aborted) return cancelled();
|
|
1341
|
+
|
|
1342
|
+
const poll = await callAccount<SpriteStatusResponse>(
|
|
1343
|
+
`/api/agent/media/sprites/${encodeURIComponent(jobId)}`,
|
|
1344
|
+
{ method: "GET", signal },
|
|
1345
|
+
);
|
|
1346
|
+
if (!poll.ok) {
|
|
1347
|
+
if (!spritePollWorthRetrying(poll.status)) {
|
|
1348
|
+
return text(`Sprite job ${jobId} could not be polled: ${poll.message}`);
|
|
1349
|
+
}
|
|
1350
|
+
if (++consecutiveFailures >= SPRITE_POLL_FAILURE_BUDGET || Date.now() > deadline) {
|
|
1351
|
+
return text(
|
|
1352
|
+
`Sprite job ${jobId} could not be polled ${consecutiveFailures} times in a row: ${poll.message}. ` +
|
|
1353
|
+
`The clips are still rendering and are still billed. ${resumeHint}`,
|
|
1354
|
+
);
|
|
1355
|
+
}
|
|
1356
|
+
continue;
|
|
1357
|
+
}
|
|
1358
|
+
consecutiveFailures = 0;
|
|
1359
|
+
|
|
1360
|
+
const status = String(poll.data.status ?? "").toLowerCase();
|
|
1361
|
+
if (status === "failed") {
|
|
1362
|
+
return text(`Sprite generation failed: ${poll.data.error?.message ?? poll.data.message ?? "the provider reported a failure"}.`);
|
|
1363
|
+
}
|
|
1364
|
+
if (status === "completed") {
|
|
1365
|
+
if (!poll.data.zip_base64) {
|
|
1366
|
+
return text(`Sprite job ${jobId} already delivered its bytes on an earlier poll; they were not saved. Generate again if the files are missing.`);
|
|
1367
|
+
}
|
|
1368
|
+
const destination = abs(cwd, dir);
|
|
1369
|
+
const bundle = Buffer.from(poll.data.zip_base64, "base64");
|
|
1370
|
+
// Keep the bytes BEFORE touching them. This poll is the only delivery the
|
|
1371
|
+
// job will ever make — the server stores nothing for the agent path and
|
|
1372
|
+
// the next poll answers "already delivered" — so anything that throws
|
|
1373
|
+
// between here and the last writeFileSync used to destroy up to five
|
|
1374
|
+
// billed clips with no way back. The copy costs a few hundred kilobytes
|
|
1375
|
+
// in the temp directory and is removed the moment the unpack succeeds.
|
|
1376
|
+
const stash = stashBundle(bundle, jobId);
|
|
1377
|
+
let written: string[];
|
|
1378
|
+
try {
|
|
1379
|
+
written = extractStoredZip(bundle, destination);
|
|
1380
|
+
} catch (e) {
|
|
1381
|
+
const why = e instanceof Error ? e.message : String(e);
|
|
1382
|
+
return text(
|
|
1383
|
+
`Sprite job ${jobId} rendered but the bundle could not be unpacked into ${destination}: ${why}. ` +
|
|
1384
|
+
(stash
|
|
1385
|
+
? `The archive itself was saved to ${stash} — unzip it there; the clips are already paid for and will not be delivered again.`
|
|
1386
|
+
: "The archive could not be saved either, so the clips are lost."),
|
|
1387
|
+
);
|
|
1388
|
+
}
|
|
1389
|
+
if (stash) {
|
|
1390
|
+
try {
|
|
1391
|
+
unlinkSync(stash);
|
|
1392
|
+
} catch {
|
|
1393
|
+
// Cleaning up a copy nobody needs is not worth failing a good unpack over.
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
const tres = written.find((f) => f.endsWith(".tres"));
|
|
1398
|
+
const anims = poll.data.animations ?? [];
|
|
1399
|
+
const mirrored = anims.filter((a) => a.origin === "mirrored").length;
|
|
1400
|
+
const sheet = poll.data.sheet;
|
|
1401
|
+
const missing = expectedAnimations != null ? expectedAnimations - anims.length : 0;
|
|
1402
|
+
|
|
1403
|
+
const lines = [
|
|
1404
|
+
`Generated sprite animation: ${written.length} files in ${destination}`,
|
|
1405
|
+
sheet ? `Sheet ${sheet.width}x${sheet.height}px, ${sheet.frame_width}x${sheet.frame_height} cells, ${sheet.columns}x${sheet.rows} grid.` : "",
|
|
1406
|
+
anims.length ? `Animations: ${anims.map((a) => a.name).join(", ")}${mirrored ? ` (${mirrored} mirrored, not billed)` : ""}.` : "",
|
|
1407
|
+
// Said out loud rather than left to be counted: a sheet short a facing is
|
|
1408
|
+
// not the sheet that was asked for, and the animation GDScript plays for
|
|
1409
|
+
// that direction is simply not in the resource.
|
|
1410
|
+
missing > 0
|
|
1411
|
+
? `WARNING: ${missing} of ${expectedAnimations} facings did not render and are NOT in the sheet or the .tres — playing them will fail in Godot. ` +
|
|
1412
|
+
"The facings that did land were billed. Re-run to try for the missing ones."
|
|
1413
|
+
: "",
|
|
1414
|
+
tres ? `Set an AnimatedSprite2D's Sprite Frames to ${poll.data.res_path ?? "res://"}${tres.split("/").pop()}.` : "",
|
|
1415
|
+
"Set the sheet's texture Filter to Nearest in the Import dock, or the pixel art imports blurry.",
|
|
1416
|
+
// Surfaced rather than swallowed: the flat backdrop the clip was asked
|
|
1417
|
+
// for is a prompt the model can ignore, and when it does the key leaves
|
|
1418
|
+
// a rim. The caller can see it here instead of finding it in-game.
|
|
1419
|
+
poll.data.key_residue != null && poll.data.key_residue > 0.08
|
|
1420
|
+
? `NOTE: the background did not key cleanly (residue ${poll.data.key_residue.toFixed(2)}) — the frames may have a fringe. Re-run, or clean them up before shipping.`
|
|
1421
|
+
: "",
|
|
1422
|
+
].filter(Boolean);
|
|
1423
|
+
return text(lines.join("\n"));
|
|
1424
|
+
}
|
|
1425
|
+
if (Date.now() > deadline) {
|
|
1426
|
+
return text(
|
|
1427
|
+
`Sprite job ${jobId} is still ${status || "running"} after ${Math.round(SPRITE_POLL_TIMEOUT_MS / 60000)} minutes. ` +
|
|
1428
|
+
`It will still complete and still be billed; nothing was saved here. ${resumeHint}`,
|
|
1429
|
+
);
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1127
1432
|
}
|
|
1128
1433
|
|
|
1129
1434
|
export const generateSpriteToolDefinition = {
|
|
@@ -1143,13 +1448,23 @@ export const generateSpriteToolDefinition = {
|
|
|
1143
1448
|
"right-facing ones rather than rendered, which is why eight animations cost five clips and not " +
|
|
1144
1449
|
"eight. Each clip is charged at the account's video rate, so an eight-way set is genuinely " +
|
|
1145
1450
|
"expensive; say the total to the user before batching characters.\n" +
|
|
1146
|
-
"TWO models are involved, which matters when one of them is down
|
|
1147
|
-
"picture to face each direction (only for `directions` 'four'
|
|
1148
|
-
"renders the motion per facing. `image_model` and `model`
|
|
1149
|
-
"
|
|
1150
|
-
"
|
|
1151
|
-
"
|
|
1152
|
-
"
|
|
1451
|
+
"TWO models are involved, which matters when one of them is down AND when you are quoting a " +
|
|
1452
|
+
"price: an IMAGE model turns the picture to face each direction (only for `directions` 'four' " +
|
|
1453
|
+
"and 'eight'), then a VIDEO model renders the motion per facing. `image_model` and `model` " +
|
|
1454
|
+
"override them separately. The turns are billed too, at the image rate — one still per billed " +
|
|
1455
|
+
"facing after the first, so 'four' draws two and 'eight' draws four. They are small beside a " +
|
|
1456
|
+
"clip, but a total that counts only clips is short; media_capabilities reports both numbers " +
|
|
1457
|
+
"per direction set (`billedClips` and `billedTurnStills`).\n" +
|
|
1458
|
+
"It takes several minutes — the picture is turned to face each direction one at a time before " +
|
|
1459
|
+
"anything is billed, then the clips render together — and this tool waits. Frame count, cell " +
|
|
1460
|
+
"size and frame rate are chosen here and cost nothing extra.\n" +
|
|
1461
|
+
"If a call comes back saying the job is still running, or that it could not be polled, DO NOT " +
|
|
1462
|
+
"call this again with the same image and prompt — that bills a whole second fan-out. Call it " +
|
|
1463
|
+
"with `resumeJobId` set to the job id it reported (and the same `dir`) to keep waiting on the " +
|
|
1464
|
+
"clips the account has already paid for.\n" +
|
|
1465
|
+
"AVAILABILITY: this needs a video decoder on the Privateer API and some deployments do not have " +
|
|
1466
|
+
"one — call media_capabilities and check `sprites.available` before spending, or you will get a " +
|
|
1467
|
+
"clear refusal instead of a sheet. " +
|
|
1153
1468
|
"PRIVACY: video and image models have no zero-retention option, so this is gated the way 3D is — " +
|
|
1154
1469
|
"a ZDR account must have enabled non-ZDR media.",
|
|
1155
1470
|
parameters: Type.Object({
|
|
@@ -1172,6 +1487,12 @@ export const generateSpriteToolDefinition = {
|
|
|
1172
1487
|
"baked into the .tres is derived from it, so running at your Godot project root means the " +
|
|
1173
1488
|
"resource resolves with nothing to edit.",
|
|
1174
1489
|
}),
|
|
1490
|
+
resumeJobId: Type.Optional(Type.String({
|
|
1491
|
+
description:
|
|
1492
|
+
"Resume waiting on a job already submitted (from a previous call that timed out or lost its " +
|
|
1493
|
+
"poll). Nothing is generated and nothing is billed: it only polls and unpacks. `image`, " +
|
|
1494
|
+
"`prompt` and every other setting are ignored — pass the same `dir`.",
|
|
1495
|
+
})),
|
|
1175
1496
|
action: Type.Optional(
|
|
1176
1497
|
Type.String({
|
|
1177
1498
|
description:
|
|
@@ -1180,12 +1501,16 @@ export const generateSpriteToolDefinition = {
|
|
|
1180
1501
|
}),
|
|
1181
1502
|
),
|
|
1182
1503
|
directions: Type.Optional(
|
|
1183
|
-
Type.
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1504
|
+
Type.Union(
|
|
1505
|
+
[Type.Literal("one"), Type.Literal("four"), Type.Literal("eight")],
|
|
1506
|
+
{
|
|
1507
|
+
description:
|
|
1508
|
+
"'one' (default, one animation, ONE clip billed), 'four' (down/right/up/left, THREE billed) " +
|
|
1509
|
+
"or 'eight' (adds the diagonals, FIVE billed). Four is the usual choice for a top-down or " +
|
|
1510
|
+
"2.5D character; eight only if the game actually turns that finely. These three words are " +
|
|
1511
|
+
"the only accepted values — not '4', not 'four-way'.",
|
|
1512
|
+
},
|
|
1513
|
+
),
|
|
1189
1514
|
),
|
|
1190
1515
|
frames: Type.Optional(
|
|
1191
1516
|
Type.Number({
|
|
@@ -1237,7 +1562,7 @@ export const generateSpriteToolDefinition = {
|
|
|
1237
1562
|
async execute(
|
|
1238
1563
|
_toolCallId: string,
|
|
1239
1564
|
params: {
|
|
1240
|
-
image: string; prompt: string; dir: string; action?: string; directions?: string;
|
|
1565
|
+
image: string; prompt: string; dir: string; resumeJobId?: string; action?: string; directions?: string;
|
|
1241
1566
|
frames?: number; frame_size?: number; fps?: number; loop?: boolean;
|
|
1242
1567
|
name?: string; res_path?: string; model?: string; image_model?: string;
|
|
1243
1568
|
},
|
|
@@ -1246,9 +1571,36 @@ export const generateSpriteToolDefinition = {
|
|
|
1246
1571
|
ctx?: { cwd?: string },
|
|
1247
1572
|
) {
|
|
1248
1573
|
const cwd = ctx?.cwd ?? process.cwd();
|
|
1574
|
+
if (!params.dir) return text("Error: dir is required — say where to write the sheet and the .tres.");
|
|
1575
|
+
|
|
1576
|
+
// RESUME. Submit nothing, bill nothing — just go back to waiting on a fan-out
|
|
1577
|
+
// the account has already paid for. The direction set that chose the facing
|
|
1578
|
+
// count is gone with the original request, so the short-sheet check is off.
|
|
1579
|
+
const resumeJobId = String(params.resumeJobId ?? "").trim();
|
|
1580
|
+
if (resumeJobId) return awaitSpriteJob(resumeJobId, cwd, params.dir, null, null, signal);
|
|
1581
|
+
|
|
1249
1582
|
if (!params.image) return text("Error: image is required — sprite generation derives every facing from one picture.");
|
|
1250
1583
|
if (!params.prompt?.trim()) return text("Error: prompt is required — describe how the character moves.");
|
|
1251
|
-
|
|
1584
|
+
|
|
1585
|
+
// Checked rather than passed through: the server CLAMPS a number out of range
|
|
1586
|
+
// and falls back on an unknown direction set, both silently, so a typo comes
|
|
1587
|
+
// back as a sheet that is quietly not the one that was asked for — after the
|
|
1588
|
+
// clips are billed. Refusing costs nothing and names the fix.
|
|
1589
|
+
const directions = (params.directions ?? "one") as SpriteDirections;
|
|
1590
|
+
if (!Object.prototype.hasOwnProperty.call(SPRITE_DIRECTION_SETS, directions)) {
|
|
1591
|
+
return text(`Error: directions must be 'one', 'four' or 'eight' — got ${JSON.stringify(params.directions)}.`);
|
|
1592
|
+
}
|
|
1593
|
+
const ranges: [string, number | undefined, number, number][] = [
|
|
1594
|
+
["frames", params.frames, 2, 24],
|
|
1595
|
+
["frame_size", params.frame_size, 8, 512],
|
|
1596
|
+
["fps", params.fps, 1, 120],
|
|
1597
|
+
];
|
|
1598
|
+
for (const [label, value, min, max] of ranges) {
|
|
1599
|
+
if (value == null) continue;
|
|
1600
|
+
if (!Number.isFinite(value) || value < min || value > max) {
|
|
1601
|
+
return text(`Error: ${label} must be between ${min} and ${max} — got ${value}.`);
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1252
1604
|
|
|
1253
1605
|
let seed: { data: string; mimeType: string };
|
|
1254
1606
|
try {
|
|
@@ -1265,7 +1617,7 @@ export const generateSpriteToolDefinition = {
|
|
|
1265
1617
|
prompt: params.prompt,
|
|
1266
1618
|
...(params.action ? { action: params.action } : {}),
|
|
1267
1619
|
...(params.name ? { name: params.name } : {}),
|
|
1268
|
-
|
|
1620
|
+
directions,
|
|
1269
1621
|
...(params.frames != null ? { frames: params.frames } : {}),
|
|
1270
1622
|
...(params.frame_size != null ? { frame_size: params.frame_size } : {}),
|
|
1271
1623
|
...(params.fps != null ? { fps: params.fps } : {}),
|
|
@@ -1286,67 +1638,11 @@ export const generateSpriteToolDefinition = {
|
|
|
1286
1638
|
const jobId = submitted.data.id;
|
|
1287
1639
|
if (!jobId) return text("Sprite generation failed: Privateer did not return a job id.");
|
|
1288
1640
|
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
//
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
text(`Sprite job ${jobId} was submitted but the wait was cancelled. Its ${billed ?? "queued"} clip(s) are still rendering and will still be billed.`);
|
|
1295
|
-
|
|
1296
|
-
for (;;) {
|
|
1297
|
-
if (signal?.aborted) return cancelled();
|
|
1298
|
-
await sleep(SPRITE_POLL_INTERVAL_MS, signal);
|
|
1299
|
-
if (signal?.aborted) return cancelled();
|
|
1300
|
-
|
|
1301
|
-
const poll = await callAccount<SpriteStatusResponse>(
|
|
1302
|
-
`/api/agent/media/sprites/${encodeURIComponent(jobId)}`,
|
|
1303
|
-
{ method: "GET", signal },
|
|
1304
|
-
);
|
|
1305
|
-
if (!poll.ok) return text(`Sprite job ${jobId} could not be polled: ${poll.message}`);
|
|
1306
|
-
|
|
1307
|
-
const status = String(poll.data.status ?? "").toLowerCase();
|
|
1308
|
-
if (status === "failed") {
|
|
1309
|
-
return text(`Sprite generation failed: ${poll.data.error?.message ?? poll.data.message ?? "the provider reported a failure"}.`);
|
|
1310
|
-
}
|
|
1311
|
-
if (status === "completed") {
|
|
1312
|
-
if (!poll.data.zip_base64) {
|
|
1313
|
-
return text(`Sprite job ${jobId} already delivered its bytes on an earlier poll; they were not saved. Generate again if the files are missing.`);
|
|
1314
|
-
}
|
|
1315
|
-
const destination = abs(cwd, params.dir);
|
|
1316
|
-
let written: string[];
|
|
1317
|
-
try {
|
|
1318
|
-
written = extractStoredZip(Buffer.from(poll.data.zip_base64, "base64"), destination);
|
|
1319
|
-
} catch (e) {
|
|
1320
|
-
return text(`Sprite job ${jobId} rendered but the bundle could not be unpacked: ${e instanceof Error ? e.message : String(e)}`);
|
|
1321
|
-
}
|
|
1322
|
-
|
|
1323
|
-
const tres = written.find((f) => f.endsWith(".tres"));
|
|
1324
|
-
const anims = poll.data.animations ?? [];
|
|
1325
|
-
const mirrored = anims.filter((a) => a.origin === "mirrored").length;
|
|
1326
|
-
const sheet = poll.data.sheet;
|
|
1327
|
-
|
|
1328
|
-
const lines = [
|
|
1329
|
-
`Generated sprite animation: ${written.length} files in ${destination}`,
|
|
1330
|
-
sheet ? `Sheet ${sheet.width}x${sheet.height}px, ${sheet.frame_width}x${sheet.frame_height} cells, ${sheet.columns}x${sheet.rows} grid.` : "",
|
|
1331
|
-
anims.length ? `Animations: ${anims.map((a) => a.name).join(", ")}${mirrored ? ` (${mirrored} mirrored, not billed)` : ""}.` : "",
|
|
1332
|
-
tres ? `Set an AnimatedSprite2D's Sprite Frames to ${poll.data.res_path ?? "res://"}${tres.split("/").pop()}.` : "",
|
|
1333
|
-
"Set the sheet's texture Filter to Nearest in the Import dock, or the pixel art imports blurry.",
|
|
1334
|
-
// Surfaced rather than swallowed: the flat backdrop the clip was asked
|
|
1335
|
-
// for is a prompt the model can ignore, and when it does the key leaves
|
|
1336
|
-
// a rim. The caller can see it here instead of finding it in-game.
|
|
1337
|
-
poll.data.key_residue != null && poll.data.key_residue > 0.08
|
|
1338
|
-
? `NOTE: the background did not key cleanly (residue ${poll.data.key_residue.toFixed(2)}) — the frames may have a fringe. Re-run, or clean them up before shipping.`
|
|
1339
|
-
: "",
|
|
1340
|
-
].filter(Boolean);
|
|
1341
|
-
return text(lines.join("\n"));
|
|
1342
|
-
}
|
|
1343
|
-
if (Date.now() > deadline) {
|
|
1344
|
-
return text(
|
|
1345
|
-
`Sprite job ${jobId} is still ${status || "running"} after ${Math.round(SPRITE_POLL_TIMEOUT_MS / 60000)} minutes. ` +
|
|
1346
|
-
"It will still complete and still be billed; nothing was saved here.",
|
|
1347
|
-
);
|
|
1348
|
-
}
|
|
1349
|
-
}
|
|
1641
|
+
// The server's own count where it gave one, the table's where it did not —
|
|
1642
|
+
// the point of the check is to notice a SHORT sheet, so guessing high would
|
|
1643
|
+
// invent a failure and guessing low would hide one.
|
|
1644
|
+
const expected = submitted.data.animations ?? SPRITE_DIRECTION_SETS[directions].animations;
|
|
1645
|
+
return awaitSpriteJob(jobId, cwd, params.dir, expected, submitted.data.billed_facings ?? null, signal);
|
|
1350
1646
|
},
|
|
1351
1647
|
};
|
|
1352
1648
|
|