toolnexus 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -17
- package/dist/acp.d.ts +6 -0
- package/dist/acp.js +38 -16
- package/dist/agents/home.d.ts +85 -4
- package/dist/agents/home.js +347 -52
- package/dist/classifier.d.ts +11 -3
- package/dist/classifier.js +120 -3
- package/dist/client.d.ts +112 -5
- package/dist/client.js +397 -82
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp.js +13 -1
- package/dist/media.d.ts +465 -54
- package/dist/media.js +1819 -230
- package/dist/nodefetch.d.ts +1 -0
- package/dist/nodefetch.js +156 -0
- package/dist/transcript_search.d.ts +27 -0
- package/dist/transcript_search.js +165 -0
- package/dist/wire.d.ts +11 -1
- package/dist/wire.js +20 -3
- package/package.json +2 -2
- package/src/acp.ts +42 -15
- package/src/agents/home.ts +381 -53
- package/src/classifier.ts +117 -6
- package/src/client.ts +380 -64
- package/src/index.ts +1 -0
- package/src/mcp.ts +18 -1
- package/src/media.ts +2034 -271
- package/src/nodefetch.ts +143 -0
- package/src/transcript_search.ts +172 -0
- package/src/wire.ts +19 -2
package/src/media.ts
CHANGED
|
@@ -1,79 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Media generation (SPEC §7G,
|
|
3
|
-
* tools (source "media") an LLM calls. Providers are
|
|
2
|
+
* Media generation (SPEC §7G, ADRs 0036 + 0037) — one interface, two faces: typed verbs a host
|
|
3
|
+
* calls and tools (source "media") an LLM calls. Providers are **media clients**: built-in API
|
|
4
|
+
* clients (a2e, ElevenLabs, OpenAI, OpenRouter, fal), CLI clients (Grok, Codex), style clients
|
|
5
|
+
* over the JSON style engine, or any object implementing {@link MediaClient}.
|
|
4
6
|
*/
|
|
5
|
-
import {
|
|
7
|
+
import { spawn } from "node:child_process"
|
|
8
|
+
import fs from "node:fs"
|
|
9
|
+
import os from "node:os"
|
|
10
|
+
import path from "node:path"
|
|
11
|
+
import readline from "node:readline"
|
|
12
|
+
import { fileURLToPath } from "node:url"
|
|
6
13
|
import type { ContentPart } from "./content.js"
|
|
7
14
|
import type { JSONSchema, Tool, ToolContext, ToolResult } from "./types.js"
|
|
8
15
|
|
|
9
|
-
|
|
10
|
-
export type MediaKind = "image" | "video" | "audio" | "text"
|
|
11
|
-
|
|
12
|
-
export interface MediaVerbStyle {
|
|
13
|
-
mode: "sync" | "async"
|
|
14
|
-
kind: MediaKind
|
|
15
|
-
start: { method?: string; path: string }
|
|
16
|
-
defaults?: Record<string, unknown>
|
|
17
|
-
args: Record<string, string>
|
|
18
|
-
idPath?: string
|
|
19
|
-
poll?: {
|
|
20
|
-
method?: string
|
|
21
|
-
path: string
|
|
22
|
-
statusPath: string
|
|
23
|
-
done?: string[]
|
|
24
|
-
failed?: string[]
|
|
25
|
-
errorPath?: string
|
|
26
|
-
intervalMs?: number
|
|
27
|
-
timeoutMs?: number
|
|
28
|
-
}
|
|
29
|
-
resultPath: string
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
export interface MediaStyle {
|
|
33
|
-
baseUrl: string
|
|
34
|
-
headers?: Record<string, string>
|
|
35
|
-
errorEnvelope?: { codePath: string; okValues: unknown[]; messagePath?: string }
|
|
36
|
-
verbs: Partial<Record<MediaVerb, MediaVerbStyle>>
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export interface MediaFile {
|
|
40
|
-
type: "image" | "audio" | "file"
|
|
41
|
-
mimeType: string
|
|
42
|
-
url: string
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
export interface MediaResult {
|
|
46
|
-
files: MediaFile[]
|
|
47
|
-
text?: string
|
|
48
|
-
jobId?: string
|
|
49
|
-
status: string
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export interface MediaProgress {
|
|
53
|
-
verb: MediaVerb
|
|
54
|
-
jobId: string
|
|
55
|
-
status: string
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
export interface MediaOptions {
|
|
59
|
-
onProgress?: (p: MediaProgress) => void
|
|
60
|
-
/** HTTP transport (same seam as the LLM client's `fetch`). Omit ⇒ global fetch. */
|
|
61
|
-
fetch?: typeof fetch
|
|
62
|
-
}
|
|
16
|
+
// ---- verbs -----------------------------------------------------------------------------------
|
|
63
17
|
|
|
64
|
-
|
|
65
|
-
export
|
|
66
|
-
/** Top-level keys overwrite the request body last. Never exposed on the tool face. */
|
|
67
|
-
providerOptions?: Record<string, unknown>
|
|
68
|
-
signal?: AbortSignal
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export type ImageArgs = { prompt: string; width?: number; height?: number; aspectRatio?: string; seed?: number }
|
|
72
|
-
export type VideoArgs = { prompt: string; image?: string; seconds?: number; aspectRatio?: string }
|
|
73
|
-
export type SpeechArgs = { text: string; voice?: string }
|
|
74
|
-
export type TranscribeArgs = { audio: string }
|
|
75
|
-
export type CloneVoiceArgs = { name: string; samples: string[] }
|
|
76
|
-
export type TalkingArgs = { image: string; audio: string }
|
|
18
|
+
export type MediaVerb = "image" | "video" | "speech" | "music" | "transcribe" | "cloneVoice" | "talk"
|
|
19
|
+
export type MediaKind = "image" | "video" | "audio" | "text"
|
|
77
20
|
|
|
78
21
|
export interface MediaVerbArg {
|
|
79
22
|
name: string
|
|
@@ -127,7 +70,7 @@ export const MEDIA_VERBS: MediaVerbSpec[] = [
|
|
|
127
70
|
{
|
|
128
71
|
"verb": "video",
|
|
129
72
|
"tool": "generate_video",
|
|
130
|
-
"description": "Generate a short video from a prompt, optionally starting from an image
|
|
73
|
+
"description": "Generate a short video from a prompt, optionally starting from an image. Can take minutes.",
|
|
131
74
|
"args": [
|
|
132
75
|
{
|
|
133
76
|
"name": "prompt",
|
|
@@ -138,7 +81,7 @@ export const MEDIA_VERBS: MediaVerbSpec[] = [
|
|
|
138
81
|
{
|
|
139
82
|
"name": "image",
|
|
140
83
|
"type": "string",
|
|
141
|
-
"description": "URL of the starting image."
|
|
84
|
+
"description": "URL or local path of the starting image."
|
|
142
85
|
},
|
|
143
86
|
{
|
|
144
87
|
"name": "seconds",
|
|
@@ -166,27 +109,60 @@ export const MEDIA_VERBS: MediaVerbSpec[] = [
|
|
|
166
109
|
{
|
|
167
110
|
"name": "voice",
|
|
168
111
|
"type": "string",
|
|
169
|
-
"description": "Voice or
|
|
112
|
+
"description": "Voice name or id, including a cloned voice."
|
|
113
|
+
}
|
|
114
|
+
]
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"verb": "music",
|
|
118
|
+
"tool": "generate_music",
|
|
119
|
+
"description": "Generate music from a text prompt and return the audio.",
|
|
120
|
+
"args": [
|
|
121
|
+
{
|
|
122
|
+
"name": "prompt",
|
|
123
|
+
"type": "string",
|
|
124
|
+
"description": "Style, mood and instruments of the music.",
|
|
125
|
+
"required": true
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
"name": "seconds",
|
|
129
|
+
"type": "integer",
|
|
130
|
+
"description": "Length in seconds."
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
"name": "instrumental",
|
|
134
|
+
"type": "boolean",
|
|
135
|
+
"description": "True for no vocals."
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
"name": "lyrics",
|
|
139
|
+
"type": "string",
|
|
140
|
+
"description": "Lyrics to sing."
|
|
170
141
|
}
|
|
171
142
|
]
|
|
172
143
|
},
|
|
173
144
|
{
|
|
174
145
|
"verb": "transcribe",
|
|
175
146
|
"tool": "transcribe_audio",
|
|
176
|
-
"description": "Transcribe speech in an audio
|
|
147
|
+
"description": "Transcribe speech in an audio file to text.",
|
|
177
148
|
"args": [
|
|
178
149
|
{
|
|
179
150
|
"name": "audio",
|
|
180
151
|
"type": "string",
|
|
181
|
-
"description": "URL of the audio.",
|
|
152
|
+
"description": "URL or local path of the audio.",
|
|
182
153
|
"required": true
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
"name": "language",
|
|
157
|
+
"type": "string",
|
|
158
|
+
"description": "Language of the speech as an ISO 639-1 code, e.g. en."
|
|
183
159
|
}
|
|
184
160
|
]
|
|
185
161
|
},
|
|
186
162
|
{
|
|
187
163
|
"verb": "cloneVoice",
|
|
188
164
|
"tool": "clone_voice",
|
|
189
|
-
"description": "Create a cloned voice from audio
|
|
165
|
+
"description": "Create a cloned voice from audio samples and return its voice id.",
|
|
190
166
|
"args": [
|
|
191
167
|
{
|
|
192
168
|
"name": "name",
|
|
@@ -198,42 +174,524 @@ export const MEDIA_VERBS: MediaVerbSpec[] = [
|
|
|
198
174
|
"name": "samples",
|
|
199
175
|
"type": "array",
|
|
200
176
|
"items": "string",
|
|
201
|
-
"description": "URLs of audio samples of the voice.",
|
|
177
|
+
"description": "URLs or local paths of audio samples of the voice.",
|
|
202
178
|
"required": true
|
|
203
179
|
}
|
|
204
180
|
]
|
|
205
181
|
},
|
|
206
182
|
{
|
|
207
|
-
"verb": "
|
|
183
|
+
"verb": "talk",
|
|
208
184
|
"tool": "talking_video",
|
|
209
|
-
"description": "Make the person in an image speak
|
|
185
|
+
"description": "Make the person in an image speak, from speech audio or from text (lip-synced video).",
|
|
210
186
|
"args": [
|
|
211
187
|
{
|
|
212
188
|
"name": "image",
|
|
213
189
|
"type": "string",
|
|
214
|
-
"description": "URL of the face image.",
|
|
190
|
+
"description": "URL or local path of the face image.",
|
|
215
191
|
"required": true
|
|
216
192
|
},
|
|
217
193
|
{
|
|
218
194
|
"name": "audio",
|
|
219
195
|
"type": "string",
|
|
220
|
-
"description": "URL of the speech audio."
|
|
221
|
-
|
|
196
|
+
"description": "URL or local path of the speech audio."
|
|
197
|
+
},
|
|
198
|
+
{
|
|
199
|
+
"name": "text",
|
|
200
|
+
"type": "string",
|
|
201
|
+
"description": "Text to speak when no audio is given."
|
|
202
|
+
},
|
|
203
|
+
{
|
|
204
|
+
"name": "voice",
|
|
205
|
+
"type": "string",
|
|
206
|
+
"description": "Voice for the text."
|
|
222
207
|
}
|
|
223
208
|
]
|
|
224
209
|
}
|
|
225
210
|
]
|
|
226
211
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
212
|
+
const VERB_SPEC = new Map(MEDIA_VERBS.map((v) => [v.verb as string, v]))
|
|
213
|
+
const VERB_KIND: Record<MediaVerb, MediaKind> = {
|
|
214
|
+
image: "image", video: "video", speech: "audio", music: "audio", transcribe: "text", cloneVoice: "text", talk: "video",
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ---- public shapes ---------------------------------------------------------------------------
|
|
218
|
+
|
|
219
|
+
export interface MediaFile {
|
|
220
|
+
type: "image" | "audio" | "file"
|
|
221
|
+
mimeType: string
|
|
222
|
+
/** `http(s)` URL, or `file://` + `path` for a local file. */
|
|
223
|
+
url?: string
|
|
224
|
+
/** Absolute local path (download mode, or a CLI's own file in url mode). */
|
|
225
|
+
path?: string
|
|
226
|
+
/** Standard padded base64 (inline mode). */
|
|
227
|
+
data?: string
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export type MediaResultStatus = "completed" | "failed" | "timeout" | "cancelled" | "unavailable" | (string & {})
|
|
231
|
+
|
|
232
|
+
export interface MediaResult {
|
|
233
|
+
files: MediaFile[]
|
|
234
|
+
text?: string
|
|
235
|
+
jobId?: string
|
|
236
|
+
status: MediaResultStatus
|
|
237
|
+
/** Provider-reported usage only, never estimated (§7G.12 D21). */
|
|
238
|
+
usage?: MediaUsage
|
|
239
|
+
/** Who served the call: the preset id, the HTTP style's name (else "style"), or the client's name (§7G.10b). */
|
|
240
|
+
style?: string
|
|
241
|
+
/** The configured model for the verb of a loaded style, when one was set; never guessed. */
|
|
242
|
+
model?: string
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** §7G.12 D21: what the provider reported; absent fields are absent. */
|
|
246
|
+
export interface MediaUsage {
|
|
247
|
+
inputTokens?: number
|
|
248
|
+
outputTokens?: number
|
|
249
|
+
costUsd?: number
|
|
250
|
+
raw?: Record<string, unknown>
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export interface MediaProgress {
|
|
254
|
+
verb: MediaVerb
|
|
255
|
+
client: string
|
|
256
|
+
jobId?: string
|
|
257
|
+
status: string
|
|
258
|
+
elapsedMs: number
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** Per-call context (§7G.2). */
|
|
262
|
+
export interface MediaContext {
|
|
263
|
+
signal?: AbortSignal
|
|
264
|
+
/** Absolute deadline (epoch ms or Date); the job ends at the earlier of it and `loop.timeoutMs`. */
|
|
265
|
+
deadline?: number | Date
|
|
266
|
+
/** Called for every progress status; when absent the client's `loop.onProgress` is used. */
|
|
267
|
+
progress?: (p: MediaProgress) => void
|
|
268
|
+
/** Function face only: top-level keys overwrite the request body last. */
|
|
269
|
+
providerOptions?: Record<string, unknown>
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** Per-call options of the typed verbs (`media.image({ prompt, signal })`). */
|
|
273
|
+
export type MediaCallOptions = MediaContext
|
|
274
|
+
|
|
275
|
+
export interface MediaArgSpec {
|
|
276
|
+
name: string
|
|
277
|
+
required: boolean
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** Any media provider (§7G.2). Every verb is optional; `supports(verb)` is the truth. */
|
|
281
|
+
export interface MediaClient {
|
|
282
|
+
readonly name: string
|
|
283
|
+
supports(verb: MediaVerb | string): boolean
|
|
284
|
+
/** Accepted canonical args; absent ⇒ every canonical arg of the verb, canonical required. */
|
|
285
|
+
args?(verb: MediaVerb | string): MediaArgSpec[]
|
|
286
|
+
image?(args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult>
|
|
287
|
+
video?(args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult>
|
|
288
|
+
speech?(args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult>
|
|
289
|
+
music?(args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult>
|
|
290
|
+
transcribe?(args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult>
|
|
291
|
+
cloneVoice?(args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult>
|
|
292
|
+
talk?(args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult>
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
export type MediaErrorStatus = "failed" | "timeout" | "cancelled" | "unavailable"
|
|
296
|
+
|
|
297
|
+
/** A failed media verb (§7G.2): message, status and the job id when known. */
|
|
298
|
+
export class MediaError extends Error {
|
|
299
|
+
readonly status: MediaErrorStatus
|
|
300
|
+
jobId?: string
|
|
301
|
+
constructor(message: string, status: MediaErrorStatus = "failed", jobId?: string) {
|
|
302
|
+
super(message)
|
|
303
|
+
this.name = "MediaError"
|
|
304
|
+
this.status = status
|
|
305
|
+
if (jobId !== undefined) this.jobId = jobId
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** The client cannot serve the call now (no credits, quota, no key, CLI missing, ZDR …). */
|
|
310
|
+
export class MediaUnavailableError extends MediaError {
|
|
311
|
+
constructor(message: string, jobId?: string) {
|
|
312
|
+
super(message, "unavailable", jobId)
|
|
313
|
+
this.name = "MediaUnavailableError"
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
export type GatherMode = "url" | "download" | "inline"
|
|
318
|
+
|
|
319
|
+
/** One config shape for every built-in client (§7G.4). */
|
|
320
|
+
export interface MediaClientConfig {
|
|
321
|
+
/** Model of the client's primary verb (CLI clients: `-m <model>`). */
|
|
322
|
+
model?: string
|
|
323
|
+
/** Per-verb model override; wins over `model` and the default. */
|
|
324
|
+
models?: Partial<Record<MediaVerb, string>>
|
|
325
|
+
/** API key; default the client's env var, read at call time. Never logged. */
|
|
326
|
+
key?: string
|
|
327
|
+
/** Base URL override (any compatible server). */
|
|
328
|
+
url?: string
|
|
329
|
+
/** CLI argv prefix (a string is one executable, never shell-split). */
|
|
330
|
+
command?: string | string[]
|
|
331
|
+
loop?: { intervalMs?: number; timeoutMs?: number; onProgress?: (p: MediaProgress) => void }
|
|
332
|
+
/**
|
|
333
|
+
* `name`, when set, names saved files `<name><ext>`, then `<name>-2<ext>`, … (first free name);
|
|
334
|
+
* sanitised to `[A-Za-z0-9._-]`, leading `.`/`-` dropped; empty ⇒ the default naming (§7G.10b).
|
|
335
|
+
*/
|
|
336
|
+
gather?: { mode?: GatherMode; dir?: string; maxBytes?: number; keep?: boolean; name?: string }
|
|
337
|
+
/** HTTP transport (the same seam as the LLM client's `fetch`). Omit ⇒ global fetch. */
|
|
338
|
+
fetch?: typeof fetch
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// ---- style format ----------------------------------------------------------------------------
|
|
342
|
+
|
|
343
|
+
export interface MediaVerbStyle {
|
|
344
|
+
mode: "sync" | "async"
|
|
345
|
+
kind: MediaKind
|
|
346
|
+
start: { method?: string; path: string }
|
|
347
|
+
defaults?: Record<string, unknown>
|
|
348
|
+
args: Record<string, string>
|
|
349
|
+
required?: string[]
|
|
350
|
+
idPath?: string
|
|
351
|
+
poll?: {
|
|
352
|
+
method?: string
|
|
353
|
+
path: string
|
|
354
|
+
statusPath: string
|
|
355
|
+
done?: string[]
|
|
356
|
+
failed?: string[]
|
|
357
|
+
errorPath?: string
|
|
358
|
+
intervalMs?: number
|
|
359
|
+
timeoutMs?: number
|
|
360
|
+
}
|
|
361
|
+
resultPath: string
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
export interface MediaStyle {
|
|
365
|
+
name?: string
|
|
366
|
+
baseUrl: string
|
|
367
|
+
headers?: Record<string, string>
|
|
368
|
+
errorEnvelope?: { codePath: string; okValues: unknown[]; messagePath?: string }
|
|
369
|
+
verbs: Partial<Record<MediaVerb, MediaVerbStyle>>
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** The a2e.ai preset — `examples/media-styles/a2e.json`, embedded verbatim. */
|
|
373
|
+
export const A2E_STYLE: MediaStyle & { _doc?: string } = {
|
|
374
|
+
"name": "a2e",
|
|
375
|
+
"baseUrl": "https://api.a2e.ai/api",
|
|
376
|
+
"headers": {
|
|
377
|
+
"Authorization": "Bearer ${A2E_API_KEY}"
|
|
378
|
+
},
|
|
379
|
+
"verbs": {
|
|
380
|
+
"image": {
|
|
381
|
+
"mode": "async",
|
|
382
|
+
"kind": "image",
|
|
383
|
+
"start": {
|
|
384
|
+
"method": "POST",
|
|
385
|
+
"path": "/v1/userText2Image/start"
|
|
386
|
+
},
|
|
387
|
+
"defaults": {
|
|
388
|
+
"name": "toolnexus",
|
|
389
|
+
"model_type": "a2e"
|
|
390
|
+
},
|
|
391
|
+
"args": {
|
|
392
|
+
"prompt": "prompt",
|
|
393
|
+
"width": "width",
|
|
394
|
+
"height": "height",
|
|
395
|
+
"seed": "seed"
|
|
396
|
+
},
|
|
397
|
+
"idPath": "data.0._id",
|
|
398
|
+
"poll": {
|
|
399
|
+
"path": "/v1/userText2Image/{id}",
|
|
400
|
+
"statusPath": "data.current_status",
|
|
401
|
+
"done": [
|
|
402
|
+
"completed"
|
|
403
|
+
],
|
|
404
|
+
"failed": [
|
|
405
|
+
"failed",
|
|
406
|
+
"blocked",
|
|
407
|
+
"canceled"
|
|
408
|
+
],
|
|
409
|
+
"errorPath": "data.failed_message",
|
|
410
|
+
"intervalMs": 3000,
|
|
411
|
+
"timeoutMs": 300000
|
|
412
|
+
},
|
|
413
|
+
"resultPath": "data.image_urls"
|
|
414
|
+
},
|
|
415
|
+
"video": {
|
|
416
|
+
"mode": "async",
|
|
417
|
+
"kind": "video",
|
|
418
|
+
"start": {
|
|
419
|
+
"method": "POST",
|
|
420
|
+
"path": "/v1/userImage2Video/start"
|
|
421
|
+
},
|
|
422
|
+
"defaults": {
|
|
423
|
+
"name": "toolnexus",
|
|
424
|
+
"generation_mode": "image-to-video",
|
|
425
|
+
"negative_prompt": "blurry, distorted, low quality"
|
|
426
|
+
},
|
|
427
|
+
"args": {
|
|
428
|
+
"image": "image_url",
|
|
429
|
+
"prompt": "prompt",
|
|
430
|
+
"seconds": "video_time"
|
|
431
|
+
},
|
|
432
|
+
"idPath": "data._id",
|
|
433
|
+
"poll": {
|
|
434
|
+
"path": "/v1/userImage2Video/{id}",
|
|
435
|
+
"statusPath": "data.current_status",
|
|
436
|
+
"done": [
|
|
437
|
+
"completed"
|
|
438
|
+
],
|
|
439
|
+
"failed": [
|
|
440
|
+
"failed",
|
|
441
|
+
"blocked",
|
|
442
|
+
"canceled"
|
|
443
|
+
],
|
|
444
|
+
"errorPath": "data.failed_message",
|
|
445
|
+
"intervalMs": 5000,
|
|
446
|
+
"timeoutMs": 900000
|
|
447
|
+
},
|
|
448
|
+
"resultPath": "data.result_url"
|
|
449
|
+
},
|
|
450
|
+
"speech": {
|
|
451
|
+
"mode": "async",
|
|
452
|
+
"kind": "audio",
|
|
453
|
+
"start": {
|
|
454
|
+
"method": "POST",
|
|
455
|
+
"path": "/v1/seedAudio/start"
|
|
456
|
+
},
|
|
457
|
+
"args": {
|
|
458
|
+
"text": "text_prompt",
|
|
459
|
+
"voice": "references.0.speaker"
|
|
460
|
+
},
|
|
461
|
+
"idPath": "data._id",
|
|
462
|
+
"poll": {
|
|
463
|
+
"path": "/v1/seedAudio/{id}",
|
|
464
|
+
"statusPath": "data.current_status",
|
|
465
|
+
"done": [
|
|
466
|
+
"completed"
|
|
467
|
+
],
|
|
468
|
+
"failed": [
|
|
469
|
+
"failed"
|
|
470
|
+
],
|
|
471
|
+
"errorPath": "data.failed_message",
|
|
472
|
+
"intervalMs": 3000,
|
|
473
|
+
"timeoutMs": 300000
|
|
474
|
+
},
|
|
475
|
+
"resultPath": "data.result_url"
|
|
476
|
+
},
|
|
477
|
+
"talk": {
|
|
478
|
+
"mode": "async",
|
|
479
|
+
"kind": "video",
|
|
480
|
+
"start": {
|
|
481
|
+
"method": "POST",
|
|
482
|
+
"path": "/v1/talkingPhoto/start"
|
|
483
|
+
},
|
|
484
|
+
"defaults": {
|
|
485
|
+
"name": "toolnexus"
|
|
486
|
+
},
|
|
487
|
+
"args": {
|
|
488
|
+
"image": "image_url",
|
|
489
|
+
"audio": "audio_url"
|
|
490
|
+
},
|
|
491
|
+
"required": [
|
|
492
|
+
"audio"
|
|
493
|
+
],
|
|
494
|
+
"idPath": "data._id",
|
|
495
|
+
"poll": {
|
|
496
|
+
"path": "/v1/talkingPhoto/{id}",
|
|
497
|
+
"statusPath": "data.current_status",
|
|
498
|
+
"done": [
|
|
499
|
+
"completed"
|
|
500
|
+
],
|
|
501
|
+
"failed": [
|
|
502
|
+
"failed",
|
|
503
|
+
"blocked",
|
|
504
|
+
"canceled"
|
|
505
|
+
],
|
|
506
|
+
"errorPath": "data.failed_message",
|
|
507
|
+
"intervalMs": 5000,
|
|
508
|
+
"timeoutMs": 900000
|
|
509
|
+
},
|
|
510
|
+
"resultPath": "data.result_url"
|
|
511
|
+
}
|
|
512
|
+
},
|
|
513
|
+
"errorEnvelope": {
|
|
514
|
+
"codePath": "code",
|
|
515
|
+
"okValues": [
|
|
516
|
+
0
|
|
517
|
+
],
|
|
518
|
+
"messagePath": "message"
|
|
519
|
+
},
|
|
520
|
+
"_doc": "a2e.ai preset (ADR 0036/0037). Embedded verbatim by every port as A2EMediaClient (SPEC §7G.9). Live-verified 2026-11-02: image, video, speech. talk is implemented per a2e's docs, UNVERIFIED. Set A2E_API_KEY. Result URLs expire (3-30 days); the built-in client downloads them by default."
|
|
521
|
+
} as MediaStyle
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Built-in CLI presets for {@link loadMediaStyle} (ADR 0042): media through a locally installed,
|
|
525
|
+
* logged-in CLI on the user's own subscription, no API key. The `*Mock` twins behave the same
|
|
526
|
+
* (name, verbs, args, prompt folding, errors) but never start a process and return a fixed file.
|
|
527
|
+
*/
|
|
528
|
+
export const MediaStyle = {
|
|
529
|
+
/** The `grok` CLI: image and video. */
|
|
530
|
+
XaiGrokCli: "xai-grok-cli",
|
|
531
|
+
/** The `codex` CLI: image only. */
|
|
532
|
+
OpenAICodexCli: "openai-codex-cli",
|
|
533
|
+
/** Google's Antigravity CLI `agy`: image only. */
|
|
534
|
+
GoogleAgyCli: "google-agy-cli",
|
|
535
|
+
XaiGrokCliMock: "xai-grok-cli-mock",
|
|
536
|
+
OpenAICodexCliMock: "openai-codex-cli-mock",
|
|
537
|
+
GoogleAgyCliMock: "google-agy-cli-mock",
|
|
538
|
+
} as const
|
|
539
|
+
|
|
540
|
+
/** A built-in CLI preset name. */
|
|
541
|
+
export type MediaStylePreset = (typeof MediaStyle)[keyof typeof MediaStyle]
|
|
542
|
+
|
|
543
|
+
const CLI_PRESETS: Record<string, "grok" | "codex" | "agy"> = {
|
|
544
|
+
"xai-grok-cli": "grok", "openai-codex-cli": "codex", "google-agy-cli": "agy",
|
|
545
|
+
"xai-grok-cli-mock": "grok", "openai-codex-cli-mock": "codex", "google-agy-cli-mock": "agy",
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* A loaded style (§7G.7): an HTTP style file or a built-in CLI preset, configured and usable as a
|
|
550
|
+
* {@link MediaClient} anywhere one is accepted — `createMedia(loadMediaStyle("xai-grok-cli"))`.
|
|
551
|
+
*/
|
|
552
|
+
export class LoadedMediaStyle implements MediaClient {
|
|
553
|
+
readonly name: string
|
|
554
|
+
/** The preset name, when this is a built-in CLI preset. */
|
|
555
|
+
readonly preset?: MediaStylePreset
|
|
556
|
+
/** The parsed style, when this is an HTTP style. */
|
|
557
|
+
readonly style?: MediaStyle
|
|
558
|
+
/** The config the style was loaded with. */
|
|
559
|
+
readonly config: MediaClientConfig
|
|
560
|
+
private readonly client: MediaClient
|
|
561
|
+
|
|
562
|
+
constructor(source: MediaStyle | MediaStylePreset, config: MediaClientConfig = {}) {
|
|
563
|
+
this.config = config
|
|
564
|
+
if (typeof source === "string") {
|
|
565
|
+
const label = CLI_PRESETS[source]
|
|
566
|
+
if (label === undefined) throw new Error(`style: unknown preset "${source}"`)
|
|
567
|
+
this.preset = source
|
|
568
|
+
const mock = { mock: source.endsWith("-mock") }
|
|
569
|
+
this.client = label === "grok" ? new CliMediaClient(GROK_SPEC, config, mock)
|
|
570
|
+
: label === "codex" ? new CliMediaClient(CODEX_SPEC, config, mock)
|
|
571
|
+
: new CliMediaClient(AGY_SPEC, config, mock)
|
|
572
|
+
} else {
|
|
573
|
+
this.style = source
|
|
574
|
+
this.client = new StyleMediaClient(source, config)
|
|
575
|
+
}
|
|
576
|
+
this.name = this.client.name
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
supports(verb: MediaVerb | string): boolean { return this.client.supports(verb) }
|
|
580
|
+
|
|
581
|
+
/** Who serves calls: the preset id, else the HTTP style's name, else "style" (§7G.10b). */
|
|
582
|
+
get id(): string {
|
|
583
|
+
return this.preset ?? this.style?.name ?? "style"
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Can this style run? Cheap and side-effect free (§7G.10b): a CLI preset's command is found and
|
|
588
|
+
* its own login file exists (never read); an HTTP style's config key or header `${VAR}`s are set.
|
|
589
|
+
*/
|
|
590
|
+
available(): MediaAvailability {
|
|
591
|
+
if (this.preset === undefined) {
|
|
592
|
+
const id = this.id
|
|
593
|
+
if (this.config.key) return { installed: true, loggedIn: true, reason: `${id}: key from config` }
|
|
594
|
+
const vars = new Set<string>()
|
|
595
|
+
for (const v of Object.values(this.style?.headers ?? {})) for (const m of v.matchAll(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g)) vars.add(m[1])
|
|
596
|
+
for (const name of [...vars].sort(byCodePoint)) {
|
|
597
|
+
if (!process.env[name]) return { installed: true, loggedIn: false, reason: `${id}: ${name} is not set` }
|
|
598
|
+
}
|
|
599
|
+
return { installed: true, loggedIn: true, reason: `${id}: env present` }
|
|
600
|
+
}
|
|
601
|
+
const label = CLI_PRESETS[this.preset]
|
|
602
|
+
if (this.preset.endsWith("-mock")) {
|
|
603
|
+
if (process.env[MEDIA_MOCK_AVAILABLE_ENV] === "0") {
|
|
604
|
+
return { installed: false, loggedIn: false, reason: `${label}: mock set unavailable by ${MEDIA_MOCK_AVAILABLE_ENV}=0` }
|
|
605
|
+
}
|
|
606
|
+
return { installed: true, loggedIn: true, reason: `${label}: mock` }
|
|
607
|
+
}
|
|
608
|
+
const cmd = this.config.command
|
|
609
|
+
const bin = (Array.isArray(cmd) ? cmd[0] : cmd) || label
|
|
610
|
+
if (!findCommand(bin)) return { installed: false, loggedIn: false, reason: `${label}: ${JSON.stringify(bin)} not found on PATH` }
|
|
611
|
+
const auth = cliAuthFile(label)
|
|
612
|
+
let present = false
|
|
613
|
+
try {
|
|
614
|
+
present = !fs.statSync(auth).isDirectory()
|
|
615
|
+
} catch {
|
|
616
|
+
present = false
|
|
617
|
+
}
|
|
618
|
+
return present
|
|
619
|
+
? { installed: true, loggedIn: true, reason: `${label}: installed, logged in (${auth} present)` }
|
|
620
|
+
: { installed: true, loggedIn: false, reason: `${label}: installed, not logged in (${auth} missing)` }
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
args(verb: MediaVerb | string): MediaArgSpec[] { return acceptedArgs(this.client, verb) }
|
|
624
|
+
|
|
625
|
+
private call(verb: MediaVerb, args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult> {
|
|
626
|
+
const fn = (this.client as any)[verb] as ((a: Record<string, unknown>, x: MediaContext) => Promise<MediaResult>) | undefined
|
|
627
|
+
if (typeof fn !== "function") return Promise.reject(new MediaError(`media: no style supports "${verb}"`))
|
|
628
|
+
return fn.call(this.client, args, ctx)
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
image(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.call("image", args, ctx) }
|
|
632
|
+
video(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.call("video", args, ctx) }
|
|
633
|
+
speech(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.call("speech", args, ctx) }
|
|
634
|
+
music(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.call("music", args, ctx) }
|
|
635
|
+
transcribe(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.call("transcribe", args, ctx) }
|
|
636
|
+
cloneVoice(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.call("cloneVoice", args, ctx) }
|
|
637
|
+
talk(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.call("talk", args, ctx) }
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** The cheap pre-flight answer of {@link LoadedMediaStyle.available} (§7G.10b). */
|
|
641
|
+
export interface MediaAvailability {
|
|
642
|
+
installed: boolean
|
|
643
|
+
loggedIn: boolean
|
|
644
|
+
reason: string
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/** Set to "0", every mock preset reports unavailable. */
|
|
648
|
+
export const MEDIA_MOCK_AVAILABLE_ENV = "TOOLNEXUS_MEDIA_MOCK_AVAILABLE"
|
|
649
|
+
|
|
650
|
+
function homeDir(): string {
|
|
651
|
+
return process.env.HOME || os.homedir()
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/** The file each CLI writes when logged in (checked for existence only). */
|
|
655
|
+
function cliAuthFile(label: string): string {
|
|
656
|
+
if (label === "grok") return path.join(homeDir(), ".grok", "auth.json")
|
|
657
|
+
if (label === "codex") return path.join(process.env.CODEX_HOME || path.join(homeDir(), ".codex"), "auth.json")
|
|
658
|
+
return path.join(homeDir(), ".gemini", "antigravity-cli", "antigravity-oauth-token")
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/** A name with a path separator is checked as a file; otherwise searched on PATH (executable). */
|
|
662
|
+
function findCommand(bin: string): boolean {
|
|
663
|
+
const exec = (p: string) => {
|
|
664
|
+
try {
|
|
665
|
+
if (!fs.statSync(p).isFile()) return false
|
|
666
|
+
fs.accessSync(p, fs.constants.X_OK)
|
|
667
|
+
return true
|
|
668
|
+
} catch {
|
|
669
|
+
return false
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
if (bin.includes("/") || bin.includes(path.sep)) return exec(bin)
|
|
673
|
+
const exts = process.platform === "win32" ? (process.env.PATHEXT ?? ".EXE;.CMD;.BAT").split(";") : [""]
|
|
674
|
+
for (const dir of (process.env.PATH ?? "").split(path.delimiter)) {
|
|
675
|
+
if (dir === "") continue
|
|
676
|
+
for (const x of exts) if (exec(path.join(dir, bin + x))) return true
|
|
677
|
+
}
|
|
678
|
+
return false
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Load a style: a built-in CLI preset ({@link MediaStyle}`.XaiGrokCli`, …) or a style JSON file
|
|
683
|
+
* (throws the §7G.7 style errors). `config` configures the client behind it (model, command, loop, gather).
|
|
684
|
+
*/
|
|
685
|
+
export function loadMediaStyle(pathOrPreset: MediaStylePreset | string, config: MediaClientConfig = {}): LoadedMediaStyle {
|
|
686
|
+
if (Object.prototype.hasOwnProperty.call(CLI_PRESETS, pathOrPreset)) return new LoadedMediaStyle(pathOrPreset as MediaStylePreset, config)
|
|
687
|
+
const style = JSON.parse(fs.readFileSync(pathOrPreset, "utf8")) as MediaStyle
|
|
230
688
|
validateStyle(style)
|
|
231
|
-
return style
|
|
689
|
+
return new LoadedMediaStyle(style, config)
|
|
232
690
|
}
|
|
233
691
|
|
|
234
|
-
/** §7G Empty dotPath: an args entry mapped to "" is a style error (sorted, so deterministic). */
|
|
235
692
|
function validateStyle(style: MediaStyle): void {
|
|
236
693
|
for (const verb of Object.keys(style.verbs ?? {}).sort()) {
|
|
694
|
+
if (!VERB_SPEC.has(verb)) throw new Error(`style: unknown verb "${verb}"`)
|
|
237
695
|
const args = (style.verbs as Record<string, MediaVerbStyle>)[verb]?.args ?? {}
|
|
238
696
|
for (const key of Object.keys(args).sort()) {
|
|
239
697
|
if (args[key] === "") throw new Error(`style: empty path for argument "${key}"`)
|
|
@@ -245,10 +703,10 @@ function validateStyle(style: MediaStyle): void {
|
|
|
245
703
|
|
|
246
704
|
const isIndex = (seg: string) => /^[0-9]+$/.test(seg)
|
|
247
705
|
|
|
248
|
-
function readPath(obj: unknown,
|
|
249
|
-
if (
|
|
706
|
+
function readPath(obj: unknown, p: string): unknown {
|
|
707
|
+
if (p === "") return obj
|
|
250
708
|
let cur: any = obj
|
|
251
|
-
for (const seg of
|
|
709
|
+
for (const seg of p.split(".")) {
|
|
252
710
|
if (cur === null || cur === undefined || typeof cur !== "object") return undefined
|
|
253
711
|
if (Array.isArray(cur)) {
|
|
254
712
|
if (!isIndex(seg)) return undefined
|
|
@@ -261,8 +719,8 @@ function readPath(obj: unknown, path: string): unknown {
|
|
|
261
719
|
return cur
|
|
262
720
|
}
|
|
263
721
|
|
|
264
|
-
function writePath(obj: Record<string, unknown>,
|
|
265
|
-
const segs =
|
|
722
|
+
function writePath(obj: Record<string, unknown>, p: string, value: unknown): void {
|
|
723
|
+
const segs = p.split(".")
|
|
266
724
|
let cur: any = obj
|
|
267
725
|
for (let i = 0; i < segs.length; i++) {
|
|
268
726
|
const seg = segs[i]
|
|
@@ -273,7 +731,6 @@ function writePath(obj: Record<string, unknown>, path: string, value: unknown):
|
|
|
273
731
|
return
|
|
274
732
|
}
|
|
275
733
|
let next = cur[key]
|
|
276
|
-
// A scalar — or an array followed by a non-numeric segment — is replaced (§7G).
|
|
277
734
|
if (next === null || next === undefined || typeof next !== "object" || (Array.isArray(next) && !isIndex(segs[i + 1]))) {
|
|
278
735
|
next = isIndex(segs[i + 1]) ? [] : {}
|
|
279
736
|
cur[key] = next
|
|
@@ -284,31 +741,53 @@ function writePath(obj: Record<string, unknown>, path: string, value: unknown):
|
|
|
284
741
|
|
|
285
742
|
// ---- helpers ---------------------------------------------------------------------------------
|
|
286
743
|
|
|
287
|
-
function expandEnv(v: string): string {
|
|
288
|
-
return v.replace(/\$\{([A-Za-z0-9_]+)\}/g, (_, name) => process.env[name] ?? "")
|
|
289
|
-
}
|
|
290
|
-
|
|
291
744
|
const MIME: Record<string, string> = {
|
|
292
745
|
png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", webp: "image/webp", gif: "image/gif",
|
|
293
746
|
mp4: "video/mp4", webm: "video/webm", mov: "video/quicktime",
|
|
294
747
|
mp3: "audio/mpeg", wav: "audio/wav", ogg: "audio/ogg", m4a: "audio/mp4", flac: "audio/flac",
|
|
295
748
|
}
|
|
749
|
+
const EXT: Record<string, string> = {
|
|
750
|
+
"image/png": ".png", "image/jpeg": ".jpg", "image/webp": ".webp", "image/gif": ".gif",
|
|
751
|
+
"video/mp4": ".mp4", "video/webm": ".webm", "video/quicktime": ".mov",
|
|
752
|
+
"audio/mpeg": ".mp3", "audio/wav": ".wav", "audio/x-wav": ".wav", "audio/wave": ".wav", "audio/ogg": ".ogg",
|
|
753
|
+
"audio/mp4": ".m4a", "audio/flac": ".flac", "audio/aac": ".aac", "text/plain": ".txt",
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
/** §7G.12 D23: `toolnexus/<version> (js)`, the version read from this package's manifest. */
|
|
757
|
+
const USER_AGENT = (() => {
|
|
758
|
+
let version = "0.0.0"
|
|
759
|
+
try {
|
|
760
|
+
version = JSON.parse(fs.readFileSync(fileURLToPath(new URL("../package.json", import.meta.url)), "utf8")).version ?? version
|
|
761
|
+
} catch {
|
|
762
|
+
// keep the fallback
|
|
763
|
+
}
|
|
764
|
+
return `toolnexus/${version} (js)`
|
|
765
|
+
})()
|
|
766
|
+
|
|
767
|
+
/** §7G.12 D22: a URL path's own extension (1–5 ASCII letters/digits), lower-cased, with the dot. */
|
|
768
|
+
function urlExt(u: string): string | undefined {
|
|
769
|
+
const e = extOf(u)
|
|
770
|
+
return /^[a-z0-9]{1,5}$/.test(e) ? "." + e : undefined
|
|
771
|
+
}
|
|
296
772
|
const KIND_DEFAULT_MIME: Record<string, string> = { image: "image/png", video: "video/mp4", audio: "audio/mpeg" }
|
|
297
773
|
const KIND_FILE_TYPE: Record<string, MediaFile["type"]> = { image: "image", audio: "audio", video: "file" }
|
|
298
774
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
775
|
+
/** Extension of the last path segment of a URL or path (query/fragment stripped), lower-case. */
|
|
776
|
+
function extOf(u: string): string {
|
|
777
|
+
let p = u.split("#")[0].split("?")[0]
|
|
778
|
+
const scheme = p.indexOf("://")
|
|
302
779
|
if (scheme >= 0) {
|
|
303
|
-
const slash =
|
|
304
|
-
|
|
780
|
+
const slash = p.indexOf("/", scheme + 3)
|
|
781
|
+
p = slash >= 0 ? p.slice(slash) : ""
|
|
305
782
|
}
|
|
306
|
-
const last =
|
|
783
|
+
const last = p.slice(p.lastIndexOf("/") + 1)
|
|
307
784
|
const dot = last.lastIndexOf(".")
|
|
308
|
-
|
|
309
|
-
return MIME[ext] ?? KIND_DEFAULT_MIME[kind]
|
|
785
|
+
return dot >= 0 ? last.slice(dot + 1).toLowerCase() : ""
|
|
310
786
|
}
|
|
311
787
|
|
|
788
|
+
const mimeFromExt = (u: string): string | undefined => MIME[extOf(u)]
|
|
789
|
+
const mimeFor = (u: string, kind: MediaKind): string => mimeFromExt(u) ?? KIND_DEFAULT_MIME[kind]
|
|
790
|
+
|
|
312
791
|
function jsonEqual(a: unknown, b: unknown): boolean {
|
|
313
792
|
if (a === b) return true
|
|
314
793
|
if (typeof a !== typeof b || a === null || b === null || typeof a !== "object") return false
|
|
@@ -327,7 +806,6 @@ function renderScalar(v: unknown): string {
|
|
|
327
806
|
return JSON.stringify(v)
|
|
328
807
|
}
|
|
329
808
|
|
|
330
|
-
/** Poll status text (§7G): string as-is, numbers like the envelope code, anything else "". */
|
|
331
809
|
function statusText(v: unknown): string {
|
|
332
810
|
if (typeof v === "string") return v
|
|
333
811
|
if (typeof v === "number" && Number.isFinite(v)) return String(v)
|
|
@@ -335,24 +813,28 @@ function statusText(v: unknown): string {
|
|
|
335
813
|
}
|
|
336
814
|
|
|
337
815
|
/** RFC 3986: keep unreserved A-Z a-z 0-9 - . _ ~, %XX (uppercase) every other UTF-8 byte. */
|
|
338
|
-
function
|
|
816
|
+
function pct(s: string): string {
|
|
339
817
|
return encodeURIComponent(s).replace(/[!'()*]/g, (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase())
|
|
340
818
|
}
|
|
341
819
|
|
|
342
|
-
/**
|
|
343
|
-
|
|
344
|
-
class TooManyRedirects extends Error {}
|
|
345
|
-
|
|
346
|
-
/** Echoed error bodies keep their first 1000 code points, then "…" (§7G). */
|
|
347
|
-
function capBody(text: string): string {
|
|
820
|
+
/** First `n` Unicode code points, then "…" when truncated. */
|
|
821
|
+
function cap(text: string, n: number): string {
|
|
348
822
|
const cps = Array.from(text)
|
|
349
|
-
return cps.length <=
|
|
823
|
+
return cps.length <= n ? text : cps.slice(0, n).join("") + "…"
|
|
350
824
|
}
|
|
351
825
|
|
|
352
|
-
|
|
353
|
-
|
|
826
|
+
const positive = (v: unknown): number | undefined => (typeof v === "number" && v > 0 ? v : undefined)
|
|
827
|
+
|
|
828
|
+
const REDIRECTS = new Set([301, 302, 303, 307, 308])
|
|
829
|
+
class TooManyRedirects extends Error {}
|
|
354
830
|
|
|
355
|
-
|
|
831
|
+
function originOf(u: string): string | undefined {
|
|
832
|
+
try {
|
|
833
|
+
return new URL(u).origin
|
|
834
|
+
} catch {
|
|
835
|
+
return undefined
|
|
836
|
+
}
|
|
837
|
+
}
|
|
356
838
|
|
|
357
839
|
function abortError(signal: AbortSignal): unknown {
|
|
358
840
|
return signal.reason ?? Object.assign(new Error("The operation was aborted"), { name: "AbortError" })
|
|
@@ -381,213 +863,1454 @@ function flattenStrings(v: unknown, out: string[]): void {
|
|
|
381
863
|
}
|
|
382
864
|
}
|
|
383
865
|
|
|
384
|
-
|
|
866
|
+
const decode = (b: Uint8Array) => new TextDecoder().decode(b)
|
|
867
|
+
const b64 = (b: Uint8Array) => Buffer.from(b).toString("base64")
|
|
868
|
+
const isHttp = (u: string) => /^https?:\/\//i.test(u)
|
|
869
|
+
const deadlineMs = (d: number | Date | undefined): number | undefined => (d instanceof Date ? d.getTime() : typeof d === "number" ? d : undefined)
|
|
385
870
|
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
871
|
+
function present(args: Record<string, unknown> | undefined): Record<string, unknown> {
|
|
872
|
+
const out: Record<string, unknown> = {}
|
|
873
|
+
for (const [k, v] of Object.entries(args ?? {})) if (v !== undefined && v !== null) out[k] = v
|
|
874
|
+
return out
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
function overlay(body: Record<string, unknown>, ctx: MediaContext): Record<string, unknown> {
|
|
878
|
+
for (const [k, v] of Object.entries(ctx.providerOptions ?? {})) body[k] = v
|
|
879
|
+
return body
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// ---- HTTP job (§7G.8) ------------------------------------------------------------------------
|
|
883
|
+
|
|
884
|
+
interface HttpRes {
|
|
885
|
+
status: number
|
|
886
|
+
headers: Headers
|
|
887
|
+
bytes: Uint8Array
|
|
888
|
+
over: boolean
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
interface SendOpts {
|
|
892
|
+
json?: unknown
|
|
893
|
+
body?: Uint8Array
|
|
894
|
+
contentType?: string
|
|
895
|
+
/** false ⇒ never send client headers (downloads, input fetches). */
|
|
896
|
+
auth?: boolean
|
|
897
|
+
/** Stop reading the body at limit + 1 bytes. */
|
|
898
|
+
limit?: number
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
/** One job's HTTP: redirects by hand, the origin header rule, the per-request deadline. */
|
|
902
|
+
class HttpJob {
|
|
903
|
+
jobId?: string
|
|
904
|
+
private readonly origin: string | undefined
|
|
905
|
+
constructor(
|
|
906
|
+
readonly client: string,
|
|
907
|
+
readonly verb: string,
|
|
908
|
+
baseUrl: string,
|
|
909
|
+
private readonly headers: () => Record<string, string>,
|
|
910
|
+
private readonly doFetch: typeof fetch,
|
|
911
|
+
readonly timeoutMs: number,
|
|
912
|
+
readonly deadline: number,
|
|
913
|
+
readonly signal?: AbortSignal,
|
|
914
|
+
) {
|
|
915
|
+
this.origin = originOf(baseUrl)
|
|
391
916
|
}
|
|
392
917
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
918
|
+
timedOut(): MediaError {
|
|
919
|
+
return new MediaError(
|
|
920
|
+
this.jobId !== undefined ? `job ${this.jobId} timed out after ${this.timeoutMs}ms` : `${this.verb}: timed out after ${this.timeoutMs}ms`,
|
|
921
|
+
"timeout",
|
|
922
|
+
this.jobId,
|
|
923
|
+
)
|
|
399
924
|
}
|
|
400
925
|
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
const
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
926
|
+
async send(url: string, method: string, opts: SendOpts = {}): Promise<HttpRes> {
|
|
927
|
+
const signal = this.signal
|
|
928
|
+
if (signal?.aborted) throw abortError(signal)
|
|
929
|
+
const remaining = this.deadline - Date.now()
|
|
930
|
+
if (remaining <= 0) throw this.timedOut()
|
|
931
|
+
const ctl = new AbortController()
|
|
932
|
+
const timer = setTimeout(() => ctl.abort(), remaining)
|
|
933
|
+
const forward = () => ctl.abort()
|
|
934
|
+
signal?.addEventListener("abort", forward, { once: true })
|
|
935
|
+
let body: string | Uint8Array | undefined = opts.json !== undefined ? JSON.stringify(opts.json) : opts.body
|
|
936
|
+
let contentType = opts.json !== undefined ? "application/json" : opts.contentType
|
|
937
|
+
try {
|
|
938
|
+
for (let hops = 0; ; hops++) {
|
|
939
|
+
const h: Record<string, string> = {}
|
|
940
|
+
if (body !== undefined && contentType) h["Content-Type"] = contentType
|
|
941
|
+
if (opts.auth !== false && this.origin !== undefined && originOf(url) === this.origin) Object.assign(h, this.headers())
|
|
942
|
+
if (!Object.keys(h).some((k) => k.toLowerCase() === "user-agent")) h["User-Agent"] = USER_AGENT
|
|
943
|
+
const res = await this.doFetch(url, { method, headers: h, body: body as any, signal: ctl.signal, redirect: "manual" })
|
|
944
|
+
const loc = res.headers?.get?.("location")
|
|
945
|
+
if (REDIRECTS.has(res.status) && loc) {
|
|
946
|
+
await res.body?.cancel().catch(() => {})
|
|
947
|
+
if (hops >= 10) throw new TooManyRedirects()
|
|
948
|
+
url = new URL(loc, url).toString()
|
|
949
|
+
if (res.status <= 303 && method !== "GET" && method !== "HEAD") {
|
|
950
|
+
method = "GET"
|
|
951
|
+
body = undefined
|
|
952
|
+
contentType = undefined
|
|
953
|
+
}
|
|
954
|
+
continue
|
|
955
|
+
}
|
|
956
|
+
const { bytes, over } = await readBody(res, opts.limit)
|
|
957
|
+
return { status: res.status, headers: res.headers, bytes, over }
|
|
410
958
|
}
|
|
959
|
+
} catch (e) {
|
|
960
|
+
if (signal?.aborted) throw abortError(signal)
|
|
961
|
+
if (ctl.signal.aborted) throw this.timedOut()
|
|
962
|
+
if (e instanceof TooManyRedirects) throw new MediaError(`${this.verb}: too many redirects`, "failed", this.jobId)
|
|
963
|
+
if (e instanceof MediaError) throw e
|
|
964
|
+
throw new MediaError(`${this.verb}: ${e instanceof Error ? e.message : String(e)}`, "failed", this.jobId)
|
|
965
|
+
} finally {
|
|
966
|
+
clearTimeout(timer)
|
|
967
|
+
signal?.removeEventListener("abort", forward)
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
/** A provider API call: unavailable mapping, then any other non-2xx is `HTTP <s>: <body>`. */
|
|
972
|
+
async api(method: string, url: string, opts: SendOpts = {}): Promise<HttpRes> {
|
|
973
|
+
const res = await this.send(url, method, opts)
|
|
974
|
+
if (res.status < 200 || res.status > 299) {
|
|
975
|
+
const text = decode(res.bytes)
|
|
976
|
+
if (res.status === 402 || (res.status === 429 && /quota|credit|billing|balance/i.test(text))) {
|
|
977
|
+
throw new MediaUnavailableError(`${this.client}: HTTP ${res.status}: ${cap(text, 1000)}`, this.jobId)
|
|
978
|
+
}
|
|
979
|
+
throw new MediaError(`HTTP ${res.status}: ${cap(text, 1000)}`, "failed", this.jobId)
|
|
980
|
+
}
|
|
981
|
+
return res
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
async json(method: string, url: string, opts: SendOpts = {}): Promise<{ data: any; res: HttpRes }> {
|
|
985
|
+
const res = await this.api(method, url, opts)
|
|
986
|
+
try {
|
|
987
|
+
return { data: JSON.parse(decode(res.bytes)), res }
|
|
988
|
+
} catch {
|
|
989
|
+
throw new MediaError(`${this.verb}: response is not JSON`, "failed", this.jobId)
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
/** §7G.5 download: GET, no client headers, capped at maxBytes. */
|
|
994
|
+
async download(url: string, maxBytes: number): Promise<{ bytes: Uint8Array; contentType?: string }> {
|
|
995
|
+
const res = await this.send(url, "GET", { auth: false, limit: maxBytes })
|
|
996
|
+
if (res.status < 200 || res.status > 299) throw new MediaError(`${this.verb}: download failed: HTTP ${res.status}`, "failed", this.jobId)
|
|
997
|
+
if (res.over) throw new MediaError(`${this.verb}: download failed: file exceeds gather.maxBytes (${maxBytes} bytes)`, "failed", this.jobId)
|
|
998
|
+
return { bytes: res.bytes, contentType: res.headers.get("content-type") ?? undefined }
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
async function readBody(res: Response, limit?: number): Promise<{ bytes: Uint8Array; over: boolean }> {
|
|
1003
|
+
if (limit === undefined || !res.body) return { bytes: new Uint8Array(await res.arrayBuffer()), over: false }
|
|
1004
|
+
const reader = res.body.getReader()
|
|
1005
|
+
const chunks: Uint8Array[] = []
|
|
1006
|
+
let total = 0
|
|
1007
|
+
for (;;) {
|
|
1008
|
+
const { done, value } = await reader.read()
|
|
1009
|
+
if (done) break
|
|
1010
|
+
chunks.push(value)
|
|
1011
|
+
total += value.byteLength
|
|
1012
|
+
if (total > limit) {
|
|
1013
|
+
await reader.cancel().catch(() => {})
|
|
1014
|
+
return { bytes: new Uint8Array(0), over: true }
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
return { bytes: new Uint8Array(Buffer.concat(chunks)), over: false }
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
// ---- input references (§7G.1) ----------------------------------------------------------------
|
|
1021
|
+
|
|
1022
|
+
interface InputFile {
|
|
1023
|
+
bytes: Uint8Array
|
|
1024
|
+
filename: string
|
|
1025
|
+
contentType: string
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
async function readInput(job: HttpJob, value: string, maxBytes: number): Promise<InputFile> {
|
|
1029
|
+
const fail = (reason: string) => new MediaError(`${job.verb}: cannot read "${value}": ${reason}`, "failed")
|
|
1030
|
+
let bytes: Uint8Array
|
|
1031
|
+
let segPath: string
|
|
1032
|
+
if (isHttp(value)) {
|
|
1033
|
+
const res = await job.send(value, "GET", { auth: false, limit: maxBytes })
|
|
1034
|
+
if (res.status < 200 || res.status > 299) throw fail(`HTTP ${res.status}`)
|
|
1035
|
+
if (res.over) throw fail(`file exceeds gather.maxBytes (${maxBytes} bytes)`)
|
|
1036
|
+
bytes = res.bytes
|
|
1037
|
+
segPath = new URL(value).pathname
|
|
1038
|
+
} else {
|
|
1039
|
+
let file = value
|
|
1040
|
+
try {
|
|
1041
|
+
if (/^file:\/\//i.test(value)) file = fileURLToPath(value)
|
|
1042
|
+
const st = fs.statSync(file)
|
|
1043
|
+
if (st.size > maxBytes) throw new Error(`file exceeds gather.maxBytes (${maxBytes} bytes)`)
|
|
1044
|
+
bytes = new Uint8Array(fs.readFileSync(file))
|
|
1045
|
+
} catch (e) {
|
|
1046
|
+
throw fail(e instanceof Error ? e.message : String(e))
|
|
411
1047
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
1048
|
+
segPath = file
|
|
1049
|
+
}
|
|
1050
|
+
const seg = segPath.split("?")[0].split("/").pop() ?? ""
|
|
1051
|
+
const filename = seg === "" ? "input" : seg
|
|
1052
|
+
return { bytes, filename, contentType: mimeFromExt(filename) ?? "application/octet-stream" }
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
// ---- gather (§7G.5) --------------------------------------------------------------------------
|
|
1056
|
+
|
|
1057
|
+
type RawResult =
|
|
1058
|
+
| { bytes: Uint8Array; mimeType: string }
|
|
1059
|
+
| { base64: string; mimeType: string }
|
|
1060
|
+
| { dataUrl: string }
|
|
1061
|
+
| { url: string }
|
|
1062
|
+
| { path: string }
|
|
1063
|
+
|
|
1064
|
+
interface GatherCfg {
|
|
1065
|
+
mode: GatherMode
|
|
1066
|
+
dir: string
|
|
1067
|
+
maxBytes: number
|
|
1068
|
+
keep: boolean
|
|
1069
|
+
slug: string
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
/** gather.name sanitised (§7G.10b). */
|
|
1073
|
+
function mediaSlug(name: string | undefined): string {
|
|
1074
|
+
return Buffer.from(name ?? "", "utf8").toString("latin1").replace(/[^A-Za-z0-9._-]/g, "-").replace(/^[.-]+/, "")
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
function gatherCfg(config: MediaClientConfig, dflt: GatherMode): GatherCfg {
|
|
1078
|
+
const g = config.gather ?? {}
|
|
1079
|
+
return {
|
|
1080
|
+
mode: g.mode ?? dflt,
|
|
1081
|
+
dir: path.resolve(g.dir ?? path.join(os.tmpdir(), "toolnexus-media")),
|
|
1082
|
+
maxBytes: positive(g.maxBytes) ?? 104857600,
|
|
1083
|
+
keep: g.keep ?? true,
|
|
1084
|
+
slug: mediaSlug(g.name),
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
let saveSeq = 0
|
|
1089
|
+
|
|
1090
|
+
class Gatherer {
|
|
1091
|
+
constructor(
|
|
1092
|
+
readonly cfg: GatherCfg,
|
|
1093
|
+
readonly client: string,
|
|
1094
|
+
readonly verb: string,
|
|
1095
|
+
readonly kind: MediaKind,
|
|
1096
|
+
readonly job?: HttpJob,
|
|
1097
|
+
) {}
|
|
1098
|
+
|
|
1099
|
+
private tooBig(): MediaError {
|
|
1100
|
+
return new MediaError(`${this.verb}: file exceeds gather.maxBytes (${this.cfg.maxBytes} bytes)`, "failed", this.job?.jobId)
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
private keepBytes(bytes: Uint8Array, mimeType: string, ext?: string): MediaFile {
|
|
1104
|
+
if (bytes.byteLength > this.cfg.maxBytes) throw this.tooBig()
|
|
1105
|
+
const type = KIND_FILE_TYPE[this.kind]
|
|
1106
|
+
if (this.cfg.mode === "inline") return { type, mimeType, data: b64(bytes) }
|
|
1107
|
+
const file = this.target(mimeType, ext)
|
|
1108
|
+
fs.writeFileSync(file, bytes)
|
|
1109
|
+
return { type, mimeType, path: file, url: "file://" + file }
|
|
1110
|
+
}
|
|
415
1111
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
const
|
|
1112
|
+
private target(mimeType: string, ext?: string): string {
|
|
1113
|
+
fs.mkdirSync(this.cfg.dir, { recursive: true })
|
|
1114
|
+
const e = ext ?? EXT[mimeType] ?? ".bin"
|
|
1115
|
+
if (this.cfg.slug !== "") {
|
|
1116
|
+
// First free <slug><ext>, <slug>-2<ext>, …; created exclusively so concurrent gathers never collide.
|
|
1117
|
+
for (let n = 1; ; n++) {
|
|
1118
|
+
const file = path.join(this.cfg.dir, `${this.cfg.slug}${n > 1 ? `-${n}` : ""}${e}`)
|
|
1119
|
+
try {
|
|
1120
|
+
fs.closeSync(fs.openSync(file, "wx"))
|
|
1121
|
+
return file
|
|
1122
|
+
} catch (err) {
|
|
1123
|
+
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
return path.join(this.cfg.dir, `${this.client}-${this.verb}-${Date.now()}-${++saveSeq}${ext ?? EXT[mimeType] ?? ".bin"}`)
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
async one(raw: RawResult): Promise<MediaFile> {
|
|
1131
|
+
const type = KIND_FILE_TYPE[this.kind]
|
|
1132
|
+
if ("bytes" in raw) return this.keepBytes(raw.bytes, raw.mimeType)
|
|
1133
|
+
if ("base64" in raw) return this.keepBytes(new Uint8Array(Buffer.from(raw.base64, "base64")), raw.mimeType)
|
|
1134
|
+
if ("dataUrl" in raw) {
|
|
1135
|
+
const m = /^data:([^;,]*)((?:;[^;,]*)*),(.*)$/s.exec(raw.dataUrl)
|
|
1136
|
+
const mime = (m?.[1] || KIND_DEFAULT_MIME[this.kind]).toLowerCase()
|
|
1137
|
+
const isB64 = /;base64/i.test(m?.[2] ?? "")
|
|
1138
|
+
const payload = m?.[3] ?? ""
|
|
1139
|
+
const bytes = isB64 ? Buffer.from(payload, "base64") : Buffer.from(decodeURIComponent(payload), "utf8")
|
|
1140
|
+
return this.keepBytes(new Uint8Array(bytes), mime)
|
|
1141
|
+
}
|
|
1142
|
+
if ("url" in raw) {
|
|
1143
|
+
if (this.cfg.mode === "url") return { type, mimeType: mimeFor(raw.url, this.kind), url: raw.url }
|
|
1144
|
+
if (!this.job) throw new MediaError(`${this.verb}: cannot download "${raw.url}"`)
|
|
1145
|
+
const { bytes, contentType } = await this.job.download(raw.url, this.cfg.maxBytes)
|
|
1146
|
+
const ct = contentType?.split(";")[0].trim().toLowerCase()
|
|
1147
|
+
const mime = ct && ct !== "application/octet-stream" ? ct : mimeFor(raw.url, this.kind)
|
|
1148
|
+
return this.keepBytes(bytes, mime, urlExt(raw.url))
|
|
1149
|
+
}
|
|
1150
|
+
// Local path (a CLI's own file).
|
|
1151
|
+
const src = raw.path
|
|
1152
|
+
const mimeType = mimeFor(src, this.kind)
|
|
1153
|
+
if (this.cfg.mode === "url") return { type, mimeType, path: src, url: "file://" + src }
|
|
1154
|
+
if (fs.statSync(src).size > this.cfg.maxBytes) throw this.tooBig()
|
|
1155
|
+
if (this.cfg.mode === "inline") return { type, mimeType, data: fs.readFileSync(src).toString("base64") }
|
|
1156
|
+
const file = this.target(mimeType)
|
|
1157
|
+
fs.copyFileSync(src, file)
|
|
1158
|
+
if (!this.cfg.keep) fs.rmSync(src, { force: true })
|
|
1159
|
+
return { type, mimeType, path: file, url: "file://" + file }
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
async all(raws: RawResult[]): Promise<MediaFile[]> {
|
|
1163
|
+
const out: MediaFile[] = []
|
|
1164
|
+
for (const r of raws) out.push(await this.one(r))
|
|
1165
|
+
return out
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
// ---- shared per-call plumbing ----------------------------------------------------------------
|
|
1170
|
+
|
|
1171
|
+
interface CallEnv {
|
|
1172
|
+
started: number
|
|
1173
|
+
deadline: number
|
|
1174
|
+
timeoutMs: number
|
|
1175
|
+
report: (status: string, jobId?: string) => void
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
function callEnv(client: string, verb: string, config: MediaClientConfig, ctx: MediaContext, timeoutMs: number): CallEnv {
|
|
1179
|
+
const started = Date.now()
|
|
1180
|
+
const ctxDeadline = deadlineMs(ctx.deadline)
|
|
1181
|
+
const deadline = Math.min(started + timeoutMs, ctxDeadline ?? Infinity)
|
|
1182
|
+
const sink = ctx.progress ?? config.loop?.onProgress
|
|
1183
|
+
return {
|
|
1184
|
+
started,
|
|
1185
|
+
deadline,
|
|
1186
|
+
timeoutMs,
|
|
1187
|
+
report: (status, jobId) => {
|
|
1188
|
+
const p: MediaProgress = { verb: verb as MediaVerb, client, status, elapsedMs: Date.now() - started }
|
|
1189
|
+
if (jobId !== undefined) p.jobId = jobId
|
|
1190
|
+
sink?.(p)
|
|
1191
|
+
},
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
function withJobId<T>(job: { jobId?: string }, run: () => Promise<T>): Promise<T> {
|
|
1196
|
+
return run().catch((e) => {
|
|
1197
|
+
if (e instanceof MediaError && e.jobId === undefined && job.jobId !== undefined) e.jobId = job.jobId
|
|
1198
|
+
throw e
|
|
1199
|
+
})
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
// ---- style clients (§7G.7) -------------------------------------------------------------------
|
|
1203
|
+
|
|
1204
|
+
interface StyleInternals {
|
|
1205
|
+
keyEnv?: string
|
|
1206
|
+
gatherMode?: GatherMode
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
/** A media client over one JSON style (the ADR 0036 engine). Named `style.name ?? "style"`. */
|
|
1210
|
+
export class StyleMediaClient implements MediaClient {
|
|
1211
|
+
readonly name: string
|
|
1212
|
+
protected readonly style: MediaStyle
|
|
1213
|
+
protected readonly config: MediaClientConfig
|
|
1214
|
+
private readonly internals: StyleInternals
|
|
1215
|
+
|
|
1216
|
+
constructor(style: MediaStyle, config: MediaClientConfig = {}, internals: StyleInternals = {}) {
|
|
1217
|
+
validateStyle(style)
|
|
1218
|
+
this.style = style
|
|
1219
|
+
this.config = config
|
|
1220
|
+
this.internals = internals
|
|
1221
|
+
this.name = style.name ?? "style"
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
supports(verb: string): boolean {
|
|
1225
|
+
return Object.prototype.hasOwnProperty.call(this.style.verbs ?? {}, verb) && VERB_SPEC.has(verb)
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
args(verb: string): MediaArgSpec[] {
|
|
1229
|
+
const spec = (this.style.verbs as Record<string, MediaVerbStyle | undefined>)[verb]
|
|
1230
|
+
const canon = VERB_SPEC.get(verb)
|
|
1231
|
+
if (!spec || !canon) return []
|
|
1232
|
+
const extra = new Set(spec.required ?? [])
|
|
1233
|
+
return canon.args
|
|
1234
|
+
.filter((a) => Object.prototype.hasOwnProperty.call(spec.args, a.name))
|
|
1235
|
+
.map((a) => ({ name: a.name, required: a.required === true || extra.has(a.name) }))
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
image(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("image", args, ctx) }
|
|
1239
|
+
video(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("video", args, ctx) }
|
|
1240
|
+
speech(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("speech", args, ctx) }
|
|
1241
|
+
music(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("music", args, ctx) }
|
|
1242
|
+
transcribe(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("transcribe", args, ctx) }
|
|
1243
|
+
cloneVoice(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("cloneVoice", args, ctx) }
|
|
1244
|
+
talk(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("talk", args, ctx) }
|
|
1245
|
+
|
|
1246
|
+
/** Hook for subclasses: the request body's defaults for a verb. */
|
|
1247
|
+
protected defaults(_verb: MediaVerb, spec: MediaVerbStyle): Record<string, unknown> {
|
|
1248
|
+
return spec.defaults ? structuredClone(spec.defaults) : {}
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
protected baseUrl(): string {
|
|
1252
|
+
return (this.config.url ?? this.style.baseUrl).replace(/\/+$/, "")
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
private async run(verb: MediaVerb, rawArgs: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult> {
|
|
1256
|
+
const style = this.style
|
|
1257
|
+
const spec = (style.verbs as Record<string, MediaVerbStyle | undefined>)[verb]
|
|
1258
|
+
if (!spec) throw new MediaError(`media: no client supports "${verb}"`)
|
|
1259
|
+
let key = this.config.key
|
|
1260
|
+
if (this.internals.keyEnv) {
|
|
1261
|
+
key = this.config.key || process.env[this.internals.keyEnv]
|
|
1262
|
+
if (!key) throw new MediaUnavailableError(`${this.name}: no API key (set ${this.internals.keyEnv})`)
|
|
1263
|
+
}
|
|
1264
|
+
const args = present(rawArgs)
|
|
1265
|
+
const body = this.defaults(verb, spec)
|
|
1266
|
+
for (const [k, v] of Object.entries(args)) {
|
|
1267
|
+
if (Object.prototype.hasOwnProperty.call(spec.args, k)) writePath(body, spec.args[k], v)
|
|
1268
|
+
}
|
|
1269
|
+
overlay(body, ctx)
|
|
1270
|
+
|
|
1271
|
+
const headers = (): Record<string, string> => {
|
|
419
1272
|
const h: Record<string, string> = {}
|
|
420
|
-
for (const [k, v] of Object.entries(style.headers ?? {}))
|
|
1273
|
+
for (const [k, v] of Object.entries(style.headers ?? {})) {
|
|
1274
|
+
h[k] = v.replace(/\$\{([A-Za-z0-9_]+)\}/g, (_, n) => (key !== undefined ? key : process.env[n] ?? ""))
|
|
1275
|
+
}
|
|
421
1276
|
return h
|
|
422
1277
|
}
|
|
423
|
-
const
|
|
424
|
-
|
|
425
|
-
const
|
|
426
|
-
const
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
const
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
const loc = res.headers?.get?.("location")
|
|
438
|
-
if (!REDIRECTS.has(res.status) || !loc) return res
|
|
439
|
-
await res.body?.cancel().catch(() => {})
|
|
440
|
-
if (hops >= 10) throw new TooManyRedirects()
|
|
441
|
-
url = new URL(loc, url).toString()
|
|
442
|
-
if (res.status <= 303 && method !== "GET" && method !== "HEAD") {
|
|
443
|
-
method = "GET"
|
|
444
|
-
body = undefined
|
|
1278
|
+
const loop = this.config.loop ?? {}
|
|
1279
|
+
const timeoutMs = positive(loop.timeoutMs) ?? positive(spec.poll?.timeoutMs) ?? 600000
|
|
1280
|
+
const env = callEnv(this.name, verb, this.config, ctx, timeoutMs)
|
|
1281
|
+
const base = this.baseUrl()
|
|
1282
|
+
const job = new HttpJob(this.name, verb, base, headers, this.config.fetch ?? fetch, timeoutMs, env.deadline, ctx.signal)
|
|
1283
|
+
|
|
1284
|
+
const call = async (method: string, p: string, json?: unknown): Promise<unknown> => {
|
|
1285
|
+
const { data } = await job.json(method, base + p, json !== undefined ? { json } : {})
|
|
1286
|
+
const envl = style.errorEnvelope
|
|
1287
|
+
if (envl) {
|
|
1288
|
+
const code = readPath(data, envl.codePath)
|
|
1289
|
+
if (code !== undefined && !envl.okValues.some((ok) => jsonEqual(ok, code))) {
|
|
1290
|
+
const msg = envl.messagePath ? readPath(data, envl.messagePath) : undefined
|
|
1291
|
+
throw new MediaError(`API error ${renderScalar(code)}: ${typeof msg === "string" ? msg : ""}`, "failed", job.jobId)
|
|
445
1292
|
}
|
|
446
1293
|
}
|
|
1294
|
+
return data
|
|
447
1295
|
}
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
const
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
1296
|
+
|
|
1297
|
+
return withJobId(job, async () => {
|
|
1298
|
+
let final: unknown
|
|
1299
|
+
let status = "completed"
|
|
1300
|
+
const startResp = await call((spec.start.method ?? "POST").toUpperCase(), spec.start.path, body)
|
|
1301
|
+
if (spec.mode === "async") {
|
|
1302
|
+
const poll = spec.poll!
|
|
1303
|
+
const idPath = spec.idPath ?? ""
|
|
1304
|
+
const rawId = readPath(startResp, idPath)
|
|
1305
|
+
if (typeof rawId === "string" && rawId !== "") job.jobId = rawId
|
|
1306
|
+
else if (typeof rawId === "number" && Number.isInteger(rawId)) job.jobId = String(rawId)
|
|
1307
|
+
else throw new MediaError(`${verb}: no job id at ${idPath}`)
|
|
1308
|
+
const jobId = job.jobId
|
|
1309
|
+
const interval = positive(loop.intervalMs) ?? positive(poll.intervalMs) ?? 5000
|
|
1310
|
+
const pollPath = poll.path.split("{id}").join(pct(jobId))
|
|
1311
|
+
for (;;) {
|
|
1312
|
+
await sleep(interval, ctx.signal)
|
|
1313
|
+
const resp = await call((poll.method ?? "GET").toUpperCase(), pollPath)
|
|
1314
|
+
const s = statusText(readPath(resp, poll.statusPath))
|
|
1315
|
+
env.report(s, jobId)
|
|
1316
|
+
if ((poll.done ?? []).includes(s)) {
|
|
1317
|
+
final = resp
|
|
1318
|
+
status = s
|
|
1319
|
+
break
|
|
1320
|
+
}
|
|
1321
|
+
if ((poll.failed ?? []).includes(s)) {
|
|
1322
|
+
const m = poll.errorPath ? readPath(resp, poll.errorPath) : undefined
|
|
1323
|
+
throw new MediaError(typeof m === "string" && m !== "" ? `job ${jobId} ${s}: ${m}` : `job ${jobId} ${s}`, "failed", jobId)
|
|
1324
|
+
}
|
|
1325
|
+
if (Date.now() >= env.deadline) throw job.timedOut()
|
|
1326
|
+
}
|
|
1327
|
+
} else {
|
|
1328
|
+
final = startResp
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
const value = readPath(final, spec.resultPath)
|
|
1332
|
+
const result: MediaResult = { files: [], status }
|
|
1333
|
+
if (spec.kind === "text") {
|
|
1334
|
+
if (typeof value === "string") result.text = value
|
|
1335
|
+
else if (typeof value === "number") result.text = renderScalar(value)
|
|
1336
|
+
else throw new MediaError(`${verb}: no result at ${spec.resultPath}`)
|
|
1337
|
+
} else {
|
|
1338
|
+
const urls: string[] = []
|
|
1339
|
+
flattenStrings(value, urls)
|
|
1340
|
+
if (urls.length === 0) throw new MediaError(`${verb}: no result at ${spec.resultPath}`)
|
|
1341
|
+
const g = new Gatherer(gatherCfg(this.config, this.internals.gatherMode ?? "url"), this.name, verb, spec.kind, job)
|
|
1342
|
+
result.files = await g.all(urls.map((url) => ({ url })))
|
|
1343
|
+
}
|
|
1344
|
+
if (job.jobId !== undefined) result.jobId = job.jobId
|
|
1345
|
+
return result
|
|
1346
|
+
})
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
/** Make a media client from a parsed style (§7G.7). */
|
|
1351
|
+
export function styleMediaClient(style: MediaStyle | LoadedMediaStyle, config: MediaClientConfig = {}): StyleMediaClient {
|
|
1352
|
+
if (style instanceof LoadedMediaStyle) {
|
|
1353
|
+
if (!style.style) throw new Error(`style: "${style.preset}" is a CLI preset, not an HTTP style`)
|
|
1354
|
+
return new StyleMediaClient(style.style, config)
|
|
1355
|
+
}
|
|
1356
|
+
return new StyleMediaClient(style, config)
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
/** a2e.ai (§7G.9): a style client over the embedded preset, `A2E_API_KEY`, gather `download`. */
|
|
1360
|
+
export class A2EMediaClient extends StyleMediaClient {
|
|
1361
|
+
constructor(config: MediaClientConfig = {}) {
|
|
1362
|
+
super(A2E_STYLE, config, { keyEnv: "A2E_API_KEY", gatherMode: "download" })
|
|
1363
|
+
}
|
|
1364
|
+
protected override defaults(verb: MediaVerb, spec: MediaVerbStyle): Record<string, unknown> {
|
|
1365
|
+
const d = super.defaults(verb, spec)
|
|
1366
|
+
const model = this.config.models?.[verb] ?? (verb === "image" ? this.config.model : undefined)
|
|
1367
|
+
if (verb === "image" && model) d.model_type = model
|
|
1368
|
+
return d
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
// ---- built-in API clients (§7G.9) ------------------------------------------------------------
|
|
1373
|
+
|
|
1374
|
+
interface ApiDef {
|
|
1375
|
+
name: string
|
|
1376
|
+
keyEnv: string
|
|
1377
|
+
defaultUrl: string
|
|
1378
|
+
auth: (key: string) => Record<string, string>
|
|
1379
|
+
primary: MediaVerb
|
|
1380
|
+
interval: number
|
|
1381
|
+
verbs: Partial<Record<MediaVerb, MediaArgSpec[]>>
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
const req = (name: string): MediaArgSpec => ({ name, required: true })
|
|
1385
|
+
const opt = (name: string): MediaArgSpec => ({ name, required: false })
|
|
1386
|
+
|
|
1387
|
+
interface ApiCall {
|
|
1388
|
+
verb: MediaVerb
|
|
1389
|
+
base: string
|
|
1390
|
+
job: HttpJob
|
|
1391
|
+
env: CallEnv
|
|
1392
|
+
gather: Gatherer
|
|
1393
|
+
interval: number
|
|
1394
|
+
maxBytes: number
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
/** The shared HTTP base for code-defined API clients (§7G.8 rules, gather, unavailable). */
|
|
1398
|
+
export abstract class ApiMediaClient implements MediaClient {
|
|
1399
|
+
readonly name: string
|
|
1400
|
+
protected readonly config: MediaClientConfig
|
|
1401
|
+
private readonly def: ApiDef
|
|
1402
|
+
|
|
1403
|
+
protected constructor(def: ApiDef, config: MediaClientConfig = {}) {
|
|
1404
|
+
this.def = def
|
|
1405
|
+
this.name = def.name
|
|
1406
|
+
this.config = config
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
supports(verb: string): boolean {
|
|
1410
|
+
return Object.prototype.hasOwnProperty.call(this.def.verbs, verb)
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
args(verb: string): MediaArgSpec[] {
|
|
1414
|
+
return (this.def.verbs as Record<string, MediaArgSpec[] | undefined>)[verb] ?? []
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
/** The model for a verb: `models[verb]`, else `model` for the primary verb, else the default. */
|
|
1418
|
+
protected model(verb: MediaVerb, dflt: string): string {
|
|
1419
|
+
return this.config.models?.[verb] ?? (verb === this.def.primary ? this.config.model : undefined) ?? dflt
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
protected begin(verb: MediaVerb, ctx: MediaContext): ApiCall {
|
|
1423
|
+
const key = this.config.key || process.env[this.def.keyEnv]
|
|
1424
|
+
if (!key) throw new MediaUnavailableError(`${this.name}: no API key (set ${this.def.keyEnv})`)
|
|
1425
|
+
const loop = this.config.loop ?? {}
|
|
1426
|
+
const timeoutMs = positive(loop.timeoutMs) ?? 600000
|
|
1427
|
+
const env = callEnv(this.name, verb, this.config, ctx, timeoutMs)
|
|
1428
|
+
const base = (this.config.url ?? this.def.defaultUrl).replace(/\/+$/, "")
|
|
1429
|
+
const auth = this.def.auth(key)
|
|
1430
|
+
const job = new HttpJob(this.name, verb, base, () => ({ ...auth }), this.config.fetch ?? fetch, timeoutMs, env.deadline, ctx.signal)
|
|
1431
|
+
const g = gatherCfg(this.config, "download")
|
|
1432
|
+
return {
|
|
1433
|
+
verb, base, job, env,
|
|
1434
|
+
gather: new Gatherer(g, this.name, verb, VERB_KIND[verb], job),
|
|
1435
|
+
interval: positive(loop.intervalMs) ?? this.def.interval,
|
|
1436
|
+
maxBytes: g.maxBytes,
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
/** Poll until `check` says done; `check` returns the status text and an optional failure. */
|
|
1441
|
+
protected async pollUntil(c: ApiCall, ctx: MediaContext, check: () => Promise<{ status: string; done: boolean; error?: string }>): Promise<void> {
|
|
1442
|
+
for (;;) {
|
|
1443
|
+
await sleep(c.interval, ctx.signal)
|
|
1444
|
+
const r = await check()
|
|
1445
|
+
c.env.report(r.status, c.job.jobId)
|
|
1446
|
+
if (r.done) return
|
|
1447
|
+
if (r.error !== undefined) throw new MediaError(r.error, "failed", c.job.jobId)
|
|
1448
|
+
if (Date.now() >= c.env.deadline) throw c.job.timedOut()
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
protected run(c: ApiCall, fn: () => Promise<MediaResult>): Promise<MediaResult> {
|
|
1453
|
+
return withJobId(c.job, fn)
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
/** Multipart form body (fields, then files, in the given order). */
|
|
1458
|
+
type FormEntry = { name: string; value: string } | { name: string; file: InputFile }
|
|
1459
|
+
|
|
1460
|
+
function multipart(entries: FormEntry[]): { body: Uint8Array; contentType: string } {
|
|
1461
|
+
const boundary = "toolnexus-" + Math.random().toString(16).slice(2) + Date.now().toString(16)
|
|
1462
|
+
const chunks: Buffer[] = []
|
|
1463
|
+
for (const e of entries) {
|
|
1464
|
+
if ("value" in e) {
|
|
1465
|
+
chunks.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="${e.name}"\r\n\r\n${e.value}\r\n`))
|
|
1466
|
+
} else {
|
|
1467
|
+
chunks.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="${e.name}"; filename="${e.file.filename.replace(/"/g, "%22")}"\r\nContent-Type: ${e.file.contentType}\r\n\r\n`))
|
|
1468
|
+
chunks.push(Buffer.from(e.file.bytes))
|
|
1469
|
+
chunks.push(Buffer.from("\r\n"))
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
chunks.push(Buffer.from(`--${boundary}--\r\n`))
|
|
1473
|
+
return { body: new Uint8Array(Buffer.concat(chunks)), contentType: `multipart/form-data; boundary=${boundary}` }
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
/** providerOptions on a multipart body: replace a field of the same name, else append. */
|
|
1477
|
+
function overlayForm(entries: FormEntry[], ctx: MediaContext): FormEntry[] {
|
|
1478
|
+
for (const [k, v] of Object.entries(ctx.providerOptions ?? {})) {
|
|
1479
|
+
const value = typeof v === "string" ? v : JSON.stringify(v)
|
|
1480
|
+
const i = entries.findIndex((e) => e.name === k && "value" in e)
|
|
1481
|
+
if (i >= 0) entries[i] = { name: k, value }
|
|
1482
|
+
else entries.push({ name: k, value })
|
|
1483
|
+
}
|
|
1484
|
+
return entries
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
/** ElevenLabs (§7G.9): speech (live-verified), music, transcribe, cloneVoice. */
|
|
1488
|
+
export class ElevenLabsMediaClient extends ApiMediaClient {
|
|
1489
|
+
constructor(config: MediaClientConfig = {}) {
|
|
1490
|
+
super({
|
|
1491
|
+
name: "elevenlabs", keyEnv: "ELEVENLABS_API_KEY", defaultUrl: "https://api.elevenlabs.io",
|
|
1492
|
+
auth: (k) => ({ "xi-api-key": k }), primary: "speech", interval: 5000,
|
|
1493
|
+
verbs: {
|
|
1494
|
+
speech: [req("text"), opt("voice")],
|
|
1495
|
+
music: [req("prompt"), opt("seconds"), opt("instrumental")],
|
|
1496
|
+
transcribe: [req("audio"), opt("language")],
|
|
1497
|
+
cloneVoice: [req("name"), req("samples")],
|
|
1498
|
+
},
|
|
1499
|
+
}, config)
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
speech(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1503
|
+
const c = this.begin("speech", ctx)
|
|
1504
|
+
return this.run(c, async () => {
|
|
1505
|
+
let id = "JBFqnCBsd6RMkjVDRZzb"
|
|
1506
|
+
if (args.voice !== undefined && args.voice !== null) {
|
|
1507
|
+
const voice = String(args.voice)
|
|
1508
|
+
const { data } = await c.job.json("GET", `${c.base}/v2/voices?page_size=100&search=${pct(voice)}`)
|
|
1509
|
+
const voices: any[] = Array.isArray(data?.voices) ? data.voices : []
|
|
1510
|
+
const lower = voice.toLowerCase()
|
|
1511
|
+
const hit = voices.find((v) => (typeof v?.name === "string" && v.name.toLowerCase() === lower) || v?.voice_id === voice)
|
|
1512
|
+
if (hit && typeof hit.voice_id === "string") id = hit.voice_id
|
|
1513
|
+
else if (voice.length === 20 && !/\s/.test(voice)) id = voice
|
|
1514
|
+
else throw new MediaError(`speech: voice "${voice}" not found`)
|
|
1515
|
+
}
|
|
1516
|
+
const body = overlay({ text: args.text, model_id: this.model("speech", "eleven_flash_v2_5") }, ctx)
|
|
1517
|
+
const res = await c.job.api("POST", `${c.base}/v1/text-to-speech/${pct(id)}?output_format=mp3_22050_32`, { json: body })
|
|
1518
|
+
const files = [await c.gather.one({ bytes: res.bytes, mimeType: "audio/mpeg" })]
|
|
1519
|
+
const result: MediaResult = { files, status: "completed" }
|
|
1520
|
+
const rid = res.headers.get("request-id")
|
|
1521
|
+
if (rid) result.jobId = rid
|
|
1522
|
+
return result
|
|
1523
|
+
})
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
music(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1527
|
+
const c = this.begin("music", ctx)
|
|
1528
|
+
return this.run(c, async () => {
|
|
1529
|
+
const seconds = typeof args.seconds === "number" ? args.seconds : 10
|
|
1530
|
+
const body: Record<string, unknown> = { prompt: args.prompt, music_length_ms: seconds * 1000 }
|
|
1531
|
+
if (args.instrumental === true) body.force_instrumental = true
|
|
1532
|
+
overlay(body, ctx)
|
|
1533
|
+
const res = await c.job.api("POST", `${c.base}/v1/music?output_format=mp3_44100_128`, { json: body })
|
|
1534
|
+
return { files: [await c.gather.one({ bytes: res.bytes, mimeType: "audio/mpeg" })], status: "completed" }
|
|
1535
|
+
})
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
transcribe(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1539
|
+
const c = this.begin("transcribe", ctx)
|
|
1540
|
+
return this.run(c, async () => {
|
|
1541
|
+
const file = await readInput(c.job, String(args.audio), c.maxBytes)
|
|
1542
|
+
const entries: FormEntry[] = [{ name: "model_id", value: this.config.models?.transcribe ?? "scribe_v1" }]
|
|
1543
|
+
if (args.language !== undefined && args.language !== null) entries.push({ name: "language_code", value: String(args.language) })
|
|
1544
|
+
entries.push({ name: "file", file })
|
|
1545
|
+
const { data } = await c.job.json("POST", `${c.base}/v1/speech-to-text`, multipart(overlayForm(entries, ctx)))
|
|
1546
|
+
return textResult(data?.text, "transcribe", "text")
|
|
1547
|
+
})
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
cloneVoice(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1551
|
+
const c = this.begin("cloneVoice", ctx)
|
|
1552
|
+
return this.run(c, async () => {
|
|
1553
|
+
const entries: FormEntry[] = [{ name: "name", value: String(args.name) }]
|
|
1554
|
+
const samples = Array.isArray(args.samples) ? args.samples : [args.samples]
|
|
1555
|
+
for (const s of samples) entries.push({ name: "files", file: await readInput(c.job, String(s), c.maxBytes) })
|
|
1556
|
+
const { data } = await c.job.json("POST", `${c.base}/v1/voices/add`, multipart(overlayForm(entries, ctx)))
|
|
1557
|
+
return textResult(data?.voice_id, "cloneVoice", "voice_id")
|
|
1558
|
+
})
|
|
1559
|
+
}
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
function textResult(v: unknown, verb: string, at: string): MediaResult {
|
|
1563
|
+
if (typeof v === "string") return { files: [], text: v, status: "completed" }
|
|
1564
|
+
if (typeof v === "number") return { files: [], text: renderScalar(v), status: "completed" }
|
|
1565
|
+
throw new MediaError(`${verb}: no result at ${at}`)
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
const multipartOpts = (m: { body: Uint8Array; contentType: string }): SendOpts => ({ body: m.body, contentType: m.contentType })
|
|
1569
|
+
|
|
1570
|
+
/** OpenAI (§7G.9): image (live-verified), video, speech, transcribe; `url` ⇒ any compatible server. */
|
|
1571
|
+
export class OpenAIMediaClient extends ApiMediaClient {
|
|
1572
|
+
constructor(config: MediaClientConfig = {}) {
|
|
1573
|
+
super({
|
|
1574
|
+
name: "openai", keyEnv: "OPENAI_API_KEY", defaultUrl: "https://api.openai.com/v1",
|
|
1575
|
+
auth: (k) => ({ Authorization: `Bearer ${k}` }), primary: "image", interval: 5000,
|
|
1576
|
+
verbs: {
|
|
1577
|
+
image: [req("prompt"), opt("width"), opt("height")],
|
|
1578
|
+
video: [req("prompt"), opt("seconds")],
|
|
1579
|
+
speech: [req("text"), opt("voice")],
|
|
1580
|
+
transcribe: [req("audio"), opt("language")],
|
|
1581
|
+
},
|
|
1582
|
+
}, config)
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
image(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1586
|
+
const c = this.begin("image", ctx)
|
|
1587
|
+
return this.run(c, async () => {
|
|
1588
|
+
const model = this.model("image", "gpt-image-1-mini")
|
|
1589
|
+
const w = typeof args.width === "number" ? args.width : undefined
|
|
1590
|
+
const h = typeof args.height === "number" ? args.height : undefined
|
|
1591
|
+
const size = w === undefined && h === undefined ? "1024x1024" : `${w ?? h}x${h ?? w}`
|
|
1592
|
+
const body: Record<string, unknown> = { model, prompt: args.prompt, n: 1, size }
|
|
1593
|
+
if (model.startsWith("gpt-image")) body.quality = "low"
|
|
1594
|
+
const { data } = await c.job.json("POST", `${c.base}/images/generations`, { json: overlay(body, ctx) })
|
|
1595
|
+
const fmt = data?.output_format
|
|
1596
|
+
const mime = fmt === "jpeg" ? "image/jpeg" : fmt === "webp" ? "image/webp" : "image/png"
|
|
1597
|
+
const raws: RawResult[] = []
|
|
1598
|
+
for (const e of Array.isArray(data?.data) ? data.data : []) {
|
|
1599
|
+
if (typeof e?.b64_json === "string" && e.b64_json !== "") raws.push({ base64: e.b64_json, mimeType: mime })
|
|
1600
|
+
else if (typeof e?.url === "string" && e.url !== "") raws.push({ url: e.url })
|
|
1601
|
+
}
|
|
1602
|
+
if (raws.length === 0) throw new MediaError("image: no result at data")
|
|
1603
|
+
const result: MediaResult = { files: await c.gather.all(raws), status: "completed" }
|
|
1604
|
+
const usage = reportedUsage({ inputTokens: data?.usage?.input_tokens, outputTokens: data?.usage?.output_tokens })
|
|
1605
|
+
if (usage) result.usage = usage
|
|
1606
|
+
return result
|
|
1607
|
+
})
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
speech(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1611
|
+
const c = this.begin("speech", ctx)
|
|
1612
|
+
return this.run(c, async () => {
|
|
1613
|
+
const body = overlay({ model: this.model("speech", "gpt-4o-mini-tts"), input: args.text, voice: args.voice ?? "alloy" }, ctx)
|
|
1614
|
+
const res = await c.job.api("POST", `${c.base}/audio/speech`, { json: body })
|
|
1615
|
+
return { files: [await c.gather.one({ bytes: res.bytes, mimeType: "audio/mpeg" })], status: "completed" }
|
|
1616
|
+
})
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
transcribe(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1620
|
+
const c = this.begin("transcribe", ctx)
|
|
1621
|
+
return this.run(c, async () => {
|
|
1622
|
+
const file = await readInput(c.job, String(args.audio), c.maxBytes)
|
|
1623
|
+
const entries: FormEntry[] = [{ name: "model", value: this.model("transcribe", "gpt-4o-mini-transcribe") }]
|
|
1624
|
+
if (args.language !== undefined && args.language !== null) entries.push({ name: "language", value: String(args.language) })
|
|
1625
|
+
entries.push({ name: "file", file })
|
|
1626
|
+
const { data } = await c.job.json("POST", `${c.base}/audio/transcriptions`, multipartOpts(multipart(overlayForm(entries, ctx))))
|
|
1627
|
+
return textResult(data?.text, "transcribe", "text")
|
|
1628
|
+
})
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
video(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1632
|
+
const c = this.begin("video", ctx)
|
|
1633
|
+
return this.run(c, async () => {
|
|
1634
|
+
const seconds = args.seconds ?? 4
|
|
1635
|
+
const body = overlay({ model: this.model("video", "sora-2"), prompt: args.prompt, seconds: String(seconds) }, ctx)
|
|
1636
|
+
const { data } = await c.job.json("POST", `${c.base}/videos`, { json: body })
|
|
1637
|
+
const id = data?.id
|
|
1638
|
+
if (typeof id === "string" && id !== "") c.job.jobId = id
|
|
1639
|
+
else if (typeof id === "number" && Number.isInteger(id)) c.job.jobId = String(id)
|
|
1640
|
+
else throw new MediaError("video: no job id at id")
|
|
1641
|
+
const jobId = c.job.jobId
|
|
1642
|
+
await this.pollUntil(c, ctx, async () => {
|
|
1643
|
+
const { data: p } = await c.job.json("GET", `${c.base}/videos/${pct(jobId)}`)
|
|
1644
|
+
const s = statusText(p?.status)
|
|
1645
|
+
if (s === "failed") {
|
|
1646
|
+
const m = p?.error?.message
|
|
1647
|
+
return { status: s, done: false, error: typeof m === "string" && m !== "" ? `job ${jobId} failed: ${m}` : `job ${jobId} failed` }
|
|
1648
|
+
}
|
|
1649
|
+
return { status: s, done: s === "completed" }
|
|
1650
|
+
})
|
|
1651
|
+
const res = await c.job.api("GET", `${c.base}/videos/${pct(jobId)}/content`)
|
|
1652
|
+
return { files: [await c.gather.one({ bytes: res.bytes, mimeType: "video/mp4" })], jobId, status: "completed" }
|
|
1653
|
+
})
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
/** §7G.12 D21: keep only the numeric fields the provider returned; none ⇒ no usage. */
|
|
1658
|
+
function reportedUsage(fields: Partial<Record<"inputTokens" | "outputTokens" | "costUsd", unknown>>): MediaUsage | undefined {
|
|
1659
|
+
const out: MediaUsage = {}
|
|
1660
|
+
for (const [k, v] of Object.entries(fields)) if (typeof v === "number" && Number.isFinite(v)) (out as any)[k] = v
|
|
1661
|
+
return Object.keys(out).length ? out : undefined
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
/** OpenRouter (§7G.9): image through chat completions (live-verified). */
|
|
1665
|
+
export class OpenRouterMediaClient extends ApiMediaClient {
|
|
1666
|
+
constructor(config: MediaClientConfig = {}) {
|
|
1667
|
+
super({
|
|
1668
|
+
name: "openrouter", keyEnv: "OPENROUTER_API_KEY", defaultUrl: "https://openrouter.ai/api/v1",
|
|
1669
|
+
auth: (k) => ({ Authorization: `Bearer ${k}` }), primary: "image", interval: 5000,
|
|
1670
|
+
verbs: { image: [req("prompt")] },
|
|
1671
|
+
}, config)
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
image(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1675
|
+
const c = this.begin("image", ctx)
|
|
1676
|
+
return this.run(c, async () => {
|
|
1677
|
+
const body = overlay({
|
|
1678
|
+
model: this.model("image", "google/gemini-2.5-flash-image"),
|
|
1679
|
+
modalities: ["image", "text"],
|
|
1680
|
+
messages: [{ role: "user", content: args.prompt }],
|
|
1681
|
+
}, ctx)
|
|
1682
|
+
const { data } = await c.job.json("POST", `${c.base}/chat/completions`, { json: body })
|
|
1683
|
+
let text = ""
|
|
1684
|
+
const raws: RawResult[] = []
|
|
1685
|
+
for (const ch of Array.isArray(data?.choices) ? data.choices : []) {
|
|
1686
|
+
const msg = ch?.message
|
|
1687
|
+
if (typeof msg?.content === "string") text += msg.content
|
|
1688
|
+
for (const im of Array.isArray(msg?.images) ? msg.images : []) {
|
|
1689
|
+
const u = im?.image_url?.url
|
|
1690
|
+
if (typeof u !== "string" || u === "") continue
|
|
1691
|
+
raws.push(/^data:/i.test(u) ? { dataUrl: u } : { url: u })
|
|
1692
|
+
}
|
|
1693
|
+
}
|
|
1694
|
+
if (raws.length === 0) throw new MediaError(`image: no image in response${text !== "" ? ": " + cap(text, 200) : ""}`)
|
|
1695
|
+
const result: MediaResult = { files: await c.gather.all(raws), status: "completed" }
|
|
1696
|
+
if (text !== "") result.text = text
|
|
1697
|
+
if (typeof data?.id === "string" && data.id !== "") result.jobId = data.id
|
|
1698
|
+
const usage = reportedUsage({ inputTokens: data?.usage?.prompt_tokens, outputTokens: data?.usage?.completion_tokens, costUsd: data?.usage?.cost })
|
|
1699
|
+
if (usage) result.usage = usage
|
|
1700
|
+
return result
|
|
1701
|
+
})
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
/** fal (§7G.9): the queue API for any hosted model id — image (live-verified), video. */
|
|
1706
|
+
export class FalMediaClient extends ApiMediaClient {
|
|
1707
|
+
constructor(config: MediaClientConfig = {}) {
|
|
1708
|
+
super({
|
|
1709
|
+
name: "fal", keyEnv: "FAL_API_KEY", defaultUrl: "https://queue.fal.run",
|
|
1710
|
+
auth: (k) => ({ Authorization: `Key ${k}` }), primary: "image", interval: 500,
|
|
1711
|
+
verbs: {
|
|
1712
|
+
image: [req("prompt"), opt("width"), opt("height"), opt("seed")],
|
|
1713
|
+
video: [req("prompt"), opt("image"), opt("seconds"), opt("aspectRatio")],
|
|
1714
|
+
},
|
|
1715
|
+
}, config)
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
image(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1719
|
+
const model = this.model("image", "fal-ai/flux/schnell")
|
|
1720
|
+
const input: Record<string, unknown> = { prompt: args.prompt, num_images: 1 }
|
|
1721
|
+
if (model.includes("schnell")) {
|
|
1722
|
+
input.image_size = "square"
|
|
1723
|
+
input.num_inference_steps = 4
|
|
1724
|
+
}
|
|
1725
|
+
const w = typeof args.width === "number" ? args.width : undefined
|
|
1726
|
+
const h = typeof args.height === "number" ? args.height : undefined
|
|
1727
|
+
if (w !== undefined || h !== undefined) input.image_size = { width: w ?? h, height: h ?? w }
|
|
1728
|
+
if (args.seed !== undefined) input.seed = args.seed
|
|
1729
|
+
return this.queue("image", model, input, ctx)
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
video(args: Record<string, unknown>, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
1733
|
+
const model = this.config.models?.video ??
|
|
1734
|
+
(args.image !== undefined ? "fal-ai/kling-video/v2.1/standard/image-to-video" : "fal-ai/kling-video/v2.1/standard/text-to-video")
|
|
1735
|
+
const input: Record<string, unknown> = { prompt: args.prompt, duration: String(args.seconds ?? 5) }
|
|
1736
|
+
if (args.image !== undefined) input.image_url = args.image
|
|
1737
|
+
if (args.aspectRatio !== undefined) input.aspect_ratio = args.aspectRatio
|
|
1738
|
+
return this.queue("video", model, input, ctx)
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
private queue(verb: MediaVerb, model: string, input: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult> {
|
|
1742
|
+
const c = this.begin(verb, ctx)
|
|
1743
|
+
return this.run(c, async () => {
|
|
1744
|
+
const { data } = await c.job.json("POST", `${c.base}/${model.replace(/^\/+/, "")}`, { json: overlay(input, ctx) })
|
|
1745
|
+
const id = data?.request_id
|
|
1746
|
+
if (typeof id === "string" && id !== "") c.job.jobId = id
|
|
1747
|
+
else if (typeof id === "number" && Number.isInteger(id)) c.job.jobId = String(id)
|
|
1748
|
+
else throw new MediaError(`${verb}: no job id at request_id`)
|
|
1749
|
+
const jobId = c.job.jobId
|
|
1750
|
+
const statusUrl = typeof data?.status_url === "string" ? data.status_url : `${c.base}/${model}/requests/${pct(jobId)}/status`
|
|
1751
|
+
const responseUrl = typeof data?.response_url === "string" ? data.response_url : `${c.base}/${model}/requests/${pct(jobId)}`
|
|
1752
|
+
await this.pollUntil(c, ctx, async () => {
|
|
1753
|
+
const { data: p } = await c.job.json("GET", statusUrl)
|
|
1754
|
+
const s = statusText(p?.status)
|
|
1755
|
+
if (s === "COMPLETED") return { status: s, done: true }
|
|
1756
|
+
const err = p?.error
|
|
1757
|
+
if (err !== undefined && err !== null && err !== "") {
|
|
1758
|
+
return { status: s, done: false, error: `job ${jobId} failed: ${typeof err === "string" ? err : JSON.stringify(err)}` }
|
|
1759
|
+
}
|
|
1760
|
+
return { status: s, done: false }
|
|
1761
|
+
})
|
|
1762
|
+
const { data: r } = await c.job.json("GET", responseUrl)
|
|
1763
|
+
const urls: string[] = []
|
|
1764
|
+
for (const im of Array.isArray(r?.images) ? r.images : []) if (typeof im?.url === "string" && im.url !== "") urls.push(im.url)
|
|
1765
|
+
for (const k of ["image", "video", "audio", "audio_file"]) {
|
|
1766
|
+
const u = r?.[k]?.url
|
|
1767
|
+
if (typeof u === "string" && u !== "") urls.push(u)
|
|
1768
|
+
}
|
|
1769
|
+
if (urls.length === 0) throw new MediaError(`${verb}: no result in response`)
|
|
1770
|
+
return { files: await c.gather.all(urls.map((url) => ({ url }))), jobId, status: "completed" }
|
|
1771
|
+
})
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
// ---- CLI clients (§7G.10) --------------------------------------------------------------------
|
|
1776
|
+
|
|
1777
|
+
/** What one JSON line of a CLI's stdout means. */
|
|
1778
|
+
export interface CliEvent {
|
|
1779
|
+
session?: string
|
|
1780
|
+
text?: string
|
|
1781
|
+
paths?: string[]
|
|
1782
|
+
error?: string
|
|
1783
|
+
status?: string
|
|
1784
|
+
/** Provider-reported usage (§7G.12 D21). */
|
|
1785
|
+
usage?: MediaUsage
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
/** How a {@link CliMediaClient} drives one CLI. */
|
|
1789
|
+
export interface CliSpec {
|
|
1790
|
+
name: string
|
|
1791
|
+
/** Default argv prefix when `config.command` is absent. */
|
|
1792
|
+
command: string
|
|
1793
|
+
/** Prompt template per supported verb; `{prompt}` is replaced. */
|
|
1794
|
+
templates: Partial<Record<MediaVerb, string>>
|
|
1795
|
+
/** Client argv after the command, for one run. */
|
|
1796
|
+
argv(prompt: string, cwd: string, model?: string): string[]
|
|
1797
|
+
/** Fixed env added to the process env. */
|
|
1798
|
+
env?: Record<string, string>
|
|
1799
|
+
parse(obj: Record<string, unknown>, line: string): CliEvent
|
|
1800
|
+
/** A folder whose entries are candidate files once the session is known. */
|
|
1801
|
+
folder?(session: string): string | undefined
|
|
1802
|
+
/** An unavailable message for the last error, or undefined (⇒ failed). */
|
|
1803
|
+
unavailable?(error: string, verb: MediaVerb): string | undefined
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
const CLI_PATH_RE = /\/[^\s"'()[\]]+\.(png|jpe?g|webp|gif|mp4|mov|webm|mp3|wav)/gi
|
|
1807
|
+
|
|
1808
|
+
function isFile(p: string): boolean {
|
|
1809
|
+
try {
|
|
1810
|
+
return fs.statSync(p).isFile()
|
|
1811
|
+
} catch {
|
|
1812
|
+
return false
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
/** Runs a local subscription CLI once, headless, and collects the files it made (§7G.10). */
|
|
1817
|
+
export class CliMediaClient implements MediaClient {
|
|
1818
|
+
readonly name: string
|
|
1819
|
+
protected readonly config: MediaClientConfig
|
|
1820
|
+
private readonly spec: CliSpec
|
|
1821
|
+
/** Never start the CLI; write a fixed file instead (§7G CLI mock presets). */
|
|
1822
|
+
private readonly mock: boolean
|
|
1823
|
+
|
|
1824
|
+
constructor(spec: CliSpec, config: MediaClientConfig = {}, opts: { mock?: boolean } = {}) {
|
|
1825
|
+
this.spec = spec
|
|
1826
|
+
this.name = spec.name
|
|
1827
|
+
this.config = config
|
|
1828
|
+
this.mock = opts.mock === true
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
supports(verb: string): boolean {
|
|
1832
|
+
return Object.prototype.hasOwnProperty.call(this.spec.templates, verb)
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1835
|
+
args(verb: string): MediaArgSpec[] {
|
|
1836
|
+
if (!this.supports(verb)) return []
|
|
1837
|
+
return verb === "video"
|
|
1838
|
+
? [req("prompt"), opt("seconds"), opt("aspectRatio")]
|
|
1839
|
+
: [req("prompt"), opt("width"), opt("height"), opt("aspectRatio")]
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
image(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("image", args, ctx) }
|
|
1843
|
+
video(args: Record<string, unknown>, ctx: MediaContext = {}) { return this.run("video", args, ctx) }
|
|
1844
|
+
|
|
1845
|
+
private async run(verb: MediaVerb, args: Record<string, unknown>, ctx: MediaContext): Promise<MediaResult> {
|
|
1846
|
+
const template = this.spec.templates[verb]
|
|
1847
|
+
if (template === undefined) throw new MediaError(`media: no client supports "${verb}"`)
|
|
1848
|
+
const name = this.name
|
|
1849
|
+
const cmd = Array.isArray(this.config.command) ? this.config.command : [this.config.command ?? this.spec.command]
|
|
1850
|
+
const timeoutMs = positive(this.config.loop?.timeoutMs) ?? 600000
|
|
1851
|
+
const env = callEnv(name, verb, this.config, ctx, timeoutMs)
|
|
1852
|
+
if (ctx.signal?.aborted) throw abortError(ctx.signal)
|
|
1853
|
+
const prompt = template.split("{prompt}").join(foldMediaPrompt(verb, args))
|
|
1854
|
+
if (this.mock) return this.mockRun(verb, prompt)
|
|
1855
|
+
const cwd = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "toolnexus-cli-")))
|
|
1856
|
+
try {
|
|
1857
|
+
const child = spawn(cmd[0], [...cmd.slice(1), ...this.spec.argv(prompt, cwd, this.config.model)], {
|
|
1858
|
+
cwd,
|
|
1859
|
+
env: { ...process.env, ...(this.spec.env ?? {}) },
|
|
1860
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
1861
|
+
detached: process.platform !== "win32",
|
|
1862
|
+
})
|
|
1863
|
+
const started = await new Promise<boolean>((resolve) => {
|
|
1864
|
+
child.once("spawn", () => resolve(true))
|
|
1865
|
+
child.once("error", () => resolve(false))
|
|
1866
|
+
})
|
|
1867
|
+
if (!started) throw new MediaUnavailableError(`${name}: cannot start "${cmd[0]}"`)
|
|
1868
|
+
|
|
1869
|
+
let session: string | undefined
|
|
1870
|
+
let usage: MediaUsage | undefined
|
|
1871
|
+
let text = ""
|
|
1872
|
+
const paths: string[] = []
|
|
1873
|
+
let lastError: string | undefined
|
|
1874
|
+
const rl = readline.createInterface({ input: child.stdout!, crlfDelay: Infinity })
|
|
1875
|
+
rl.on("line", (line) => {
|
|
1876
|
+
let obj: unknown
|
|
1877
|
+
try {
|
|
1878
|
+
obj = JSON.parse(line)
|
|
1879
|
+
} catch {
|
|
1880
|
+
return
|
|
1881
|
+
}
|
|
1882
|
+
if (obj === null || typeof obj !== "object" || Array.isArray(obj)) return
|
|
1883
|
+
const ev = this.spec.parse(obj as Record<string, unknown>, line)
|
|
1884
|
+
if (ev.session !== undefined) session = ev.session
|
|
1885
|
+
if (ev.usage !== undefined) usage = ev.usage
|
|
1886
|
+
if (ev.text !== undefined) text += ev.text
|
|
1887
|
+
if (ev.paths) paths.push(...ev.paths)
|
|
1888
|
+
if (ev.error !== undefined) lastError = ev.error
|
|
1889
|
+
if (ev.status !== undefined) env.report(ev.status, session)
|
|
1890
|
+
})
|
|
1891
|
+
|
|
1892
|
+
let why: "timeout" | "cancelled" | undefined
|
|
1893
|
+
const kill = () => {
|
|
1894
|
+
try {
|
|
1895
|
+
if (child.pid !== undefined && process.platform !== "win32") process.kill(-child.pid, "SIGKILL")
|
|
1896
|
+
else child.kill("SIGKILL")
|
|
1897
|
+
} catch {
|
|
1898
|
+
child.kill("SIGKILL")
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
const timer = setTimeout(() => { why ??= "timeout"; kill() }, Math.max(0, env.deadline - Date.now()))
|
|
1902
|
+
const onAbort = () => { why ??= "cancelled"; kill() }
|
|
1903
|
+
ctx.signal?.addEventListener("abort", onAbort, { once: true })
|
|
1904
|
+
const rlClosed = new Promise<void>((r) => rl.once("close", () => r()))
|
|
1905
|
+
let exitCode: number
|
|
457
1906
|
try {
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
} catch (e) {
|
|
461
|
-
if (signal?.aborted) throw abortError(signal)
|
|
462
|
-
if (ctl.signal.aborted) throw timedOut()
|
|
463
|
-
if (e instanceof TooManyRedirects) throw new Error(`${verb}: too many redirects`)
|
|
464
|
-
throw new Error(`${verb}: ${e instanceof Error ? e.message : String(e)}`)
|
|
1907
|
+
exitCode = await new Promise<number>((resolve) => child.once("close", (code) => resolve(code ?? 0)))
|
|
1908
|
+
await rlClosed
|
|
465
1909
|
} finally {
|
|
466
1910
|
clearTimeout(timer)
|
|
467
|
-
signal?.removeEventListener("abort",
|
|
1911
|
+
ctx.signal?.removeEventListener("abort", onAbort)
|
|
468
1912
|
}
|
|
469
|
-
if (
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
const code = readPath(data, env.codePath)
|
|
479
|
-
if (code !== undefined && !env.okValues.some((ok) => jsonEqual(ok, code))) {
|
|
480
|
-
const msg = env.messagePath ? readPath(data, env.messagePath) : undefined
|
|
481
|
-
throw new Error(`API error ${renderScalar(code)}: ${typeof msg === "string" ? msg : ""}`)
|
|
1913
|
+
if (why === "cancelled") throw abortError(ctx.signal!)
|
|
1914
|
+
if (why === "timeout") throw new MediaError(`${name}: timed out after ${timeoutMs}ms`, "timeout", session)
|
|
1915
|
+
|
|
1916
|
+
const seen = new Set<string>()
|
|
1917
|
+
const candidates: string[] = []
|
|
1918
|
+
const add = (p: string) => {
|
|
1919
|
+
if (!seen.has(p)) {
|
|
1920
|
+
seen.add(p)
|
|
1921
|
+
candidates.push(p)
|
|
482
1922
|
}
|
|
483
1923
|
}
|
|
484
|
-
|
|
1924
|
+
paths.forEach(add)
|
|
1925
|
+
for (const m of text.matchAll(CLI_PATH_RE)) add(m[0])
|
|
1926
|
+
if (this.spec.folder && session !== undefined) {
|
|
1927
|
+
const dir = this.spec.folder(session)
|
|
1928
|
+
if (dir) {
|
|
1929
|
+
let entries: string[] = []
|
|
1930
|
+
try {
|
|
1931
|
+
entries = fs.readdirSync(dir).sort()
|
|
1932
|
+
} catch {
|
|
1933
|
+
entries = []
|
|
1934
|
+
}
|
|
1935
|
+
for (const e of entries) add(path.join(dir, e))
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
const prefix = verb === "image" ? "image/" : "video/"
|
|
1939
|
+
const kept = candidates.filter((p) => isFile(p) && (mimeFromExt(p) ?? "").startsWith(prefix))
|
|
1940
|
+
if (kept.length > 0) {
|
|
1941
|
+
const g = new Gatherer(gatherCfg(this.config, "download"), name, verb, VERB_KIND[verb])
|
|
1942
|
+
const result: MediaResult = { files: await g.all(kept.map((p) => ({ path: p }))), status: "completed" }
|
|
1943
|
+
if (text !== "") result.text = text
|
|
1944
|
+
if (session !== undefined) result.jobId = session
|
|
1945
|
+
if (usage !== undefined) result.usage = usage
|
|
1946
|
+
return result
|
|
1947
|
+
}
|
|
1948
|
+
if (lastError !== undefined) {
|
|
1949
|
+
const un = this.spec.unavailable?.(lastError, verb)
|
|
1950
|
+
if (un !== undefined) throw new MediaUnavailableError(un, session)
|
|
1951
|
+
throw new MediaError(`${name}: no ${verb}: ${cap(lastError, 400)}`, "failed", session)
|
|
1952
|
+
}
|
|
1953
|
+
throw new MediaError(
|
|
1954
|
+
`${name}: no ${verb} file in the run (exit code ${exitCode})${text !== "" ? ": " + cap(text, 300) : ""}`,
|
|
1955
|
+
"failed",
|
|
1956
|
+
session,
|
|
1957
|
+
)
|
|
1958
|
+
} finally {
|
|
1959
|
+
fs.rmSync(cwd, { recursive: true, force: true })
|
|
485
1960
|
}
|
|
1961
|
+
}
|
|
486
1962
|
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
1963
|
+
/**
|
|
1964
|
+
* The CLI mock (§7G CLI mock presets): no process. Writes `<gather.dir>/<client>-mock-<verb>.<png|mp4>`
|
|
1965
|
+
* (gather.dir default `<tmp>/toolnexus-media`) holding `toolnexus mock <verb>` and gathers it like a
|
|
1966
|
+
* real CLI file; text = the exact prompt the CLI would have received; jobId "mock".
|
|
1967
|
+
*/
|
|
1968
|
+
private async mockRun(verb: MediaVerb, prompt: string): Promise<MediaResult> {
|
|
1969
|
+
const dir = this.config.gather?.dir || path.join(os.tmpdir(), "toolnexus-media")
|
|
1970
|
+
let file: string
|
|
1971
|
+
try {
|
|
1972
|
+
fs.mkdirSync(dir, { recursive: true })
|
|
1973
|
+
file = path.resolve(dir, `${this.name}-mock-${verb}${verb === "video" ? ".mp4" : ".png"}`)
|
|
1974
|
+
fs.writeFileSync(file, `toolnexus mock ${verb}`)
|
|
1975
|
+
} catch (e) {
|
|
1976
|
+
throw new MediaError(`${this.name}: ${(e as Error).message}`)
|
|
1977
|
+
}
|
|
1978
|
+
const g = new Gatherer(gatherCfg(this.config, "download"), this.name, verb, VERB_KIND[verb])
|
|
1979
|
+
return { files: await g.all([{ path: file }]), text: prompt, jobId: "mock", status: "completed" }
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
const CLI_HINTS: Record<string, [string, string]> = {
|
|
1984
|
+
width: ["width ", " px"],
|
|
1985
|
+
height: ["height ", " px"],
|
|
1986
|
+
seconds: ["about ", " seconds long"],
|
|
1987
|
+
aspectRatio: ["aspect ratio ", ""],
|
|
1988
|
+
}
|
|
1989
|
+
|
|
1990
|
+
/**
|
|
1991
|
+
* The prompt a CLI style receives (§7G CLI hint folding): `prompt`, then the present size hints in
|
|
1992
|
+
* canonical verb order as ` (h1, h2)`; nothing appended when none is present.
|
|
1993
|
+
*/
|
|
1994
|
+
export function foldMediaPrompt(verb: MediaVerb | string, args: Record<string, unknown>): string {
|
|
1995
|
+
const prompt = args.prompt === undefined || args.prompt === null ? "" : renderScalar(args.prompt)
|
|
1996
|
+
const hints: string[] = []
|
|
1997
|
+
for (const a of VERB_SPEC.get(verb)?.args ?? []) {
|
|
1998
|
+
const h = CLI_HINTS[a.name]
|
|
1999
|
+
const v = args[a.name]
|
|
2000
|
+
if (h && v !== undefined && v !== null) hints.push(h[0] + renderScalar(v) + h[1])
|
|
2001
|
+
}
|
|
2002
|
+
return hints.length === 0 ? prompt : `${prompt} (${hints.join(", ")})`
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined)
|
|
2006
|
+
|
|
2007
|
+
const GROK_SPEC: CliSpec = {
|
|
2008
|
+
name: "grok",
|
|
2009
|
+
command: "grok",
|
|
2010
|
+
templates: {
|
|
2011
|
+
image: "Generate exactly one image with your image generation tool: {prompt}. Do not write code. Reply only with the file path.",
|
|
2012
|
+
video: "Generate exactly one short video with your video generation tool, shortest duration available: {prompt}. Do not write code. Reply only with the file path.",
|
|
2013
|
+
},
|
|
2014
|
+
argv: (prompt, cwd, model) => [
|
|
2015
|
+
"-p", prompt, "--cwd", cwd, "--output-format", "streaming-json", "--yolo", "--no-auto-update",
|
|
2016
|
+
...(model ? ["-m", model] : []),
|
|
2017
|
+
],
|
|
2018
|
+
env: { GROK_CONFIG: '{"features":{"image_gen":true,"video_gen":true}}' },
|
|
2019
|
+
parse(o) {
|
|
2020
|
+
switch (o.type) {
|
|
2021
|
+
case "text":
|
|
2022
|
+
return { text: str(o.data) ?? "" }
|
|
2023
|
+
case "tool_call":
|
|
2024
|
+
return { status: `tool ${str(o.toolName) ?? ""}` }
|
|
2025
|
+
case "tool_call_update": {
|
|
2026
|
+
const s = str(o.status)
|
|
2027
|
+
if (!s) return {}
|
|
2028
|
+
const ev: CliEvent = { status: `tool ${s}` }
|
|
2029
|
+
const raw = o.rawOutput as any
|
|
2030
|
+
if (s === "completed") {
|
|
2031
|
+
const p = raw?.path
|
|
2032
|
+
if (typeof p === "string" && p !== "") ev.paths = [p]
|
|
2033
|
+
} else if (s === "failed") {
|
|
2034
|
+
ev.error = typeof raw?.message === "string" ? raw.message : JSON.stringify(sortKeys(raw ?? null))
|
|
512
2035
|
}
|
|
513
|
-
|
|
2036
|
+
return ev
|
|
514
2037
|
}
|
|
515
|
-
|
|
516
|
-
|
|
2038
|
+
case "end": {
|
|
2039
|
+
const ev: CliEvent = { session: str(o.sessionId), status: `end ${str(o.stopReason) ?? ""}` }
|
|
2040
|
+
// D21: tokens + total_cost_usd map to canonical fields; total_cost_usd_ticks is kept raw, never converted.
|
|
2041
|
+
const u = o.usage as any
|
|
2042
|
+
const usage = reportedUsage({ inputTokens: u?.input_tokens, outputTokens: u?.output_tokens, costUsd: o.total_cost_usd }) ?? {}
|
|
2043
|
+
if (typeof o.total_cost_usd_ticks === "number" && Number.isFinite(o.total_cost_usd_ticks)) usage.raw = { total_cost_usd_ticks: o.total_cost_usd_ticks }
|
|
2044
|
+
if (Object.keys(usage).length) ev.usage = usage
|
|
2045
|
+
return ev
|
|
2046
|
+
}
|
|
2047
|
+
case "error":
|
|
2048
|
+
return { error: str(o.message) ?? JSON.stringify(o) }
|
|
2049
|
+
default:
|
|
2050
|
+
return {}
|
|
2051
|
+
}
|
|
2052
|
+
},
|
|
2053
|
+
unavailable(error, verb) {
|
|
2054
|
+
const l = error.toLowerCase()
|
|
2055
|
+
if (l.includes("zero data retention")) {
|
|
2056
|
+
return `grok: ${verb} blocked by Zero Data Retention — run /privacy and opt in, or configure ZDR video storage (docs/media#grok-video-zdr)`
|
|
517
2057
|
}
|
|
2058
|
+
if (l.includes("usage limit")) return `grok: ${cap(error, 400)}`
|
|
2059
|
+
return undefined
|
|
2060
|
+
},
|
|
2061
|
+
}
|
|
518
2062
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
2063
|
+
/** Grok CLI (§7G.10): image, video via `grok -p … --output-format streaming-json`. */
|
|
2064
|
+
export class GrokMediaClient extends CliMediaClient {
|
|
2065
|
+
constructor(config: MediaClientConfig = {}) {
|
|
2066
|
+
super(GROK_SPEC, config)
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
const CODEX_SPEC: CliSpec = {
|
|
2071
|
+
name: "codex",
|
|
2072
|
+
command: "codex",
|
|
2073
|
+
templates: {
|
|
2074
|
+
image: "Generate exactly one image with your built-in image generation tool: {prompt}. Do not write code or use scripts. Reply only with the absolute path where the image was saved.",
|
|
2075
|
+
},
|
|
2076
|
+
argv: (prompt, _cwd, model) => ["exec", "--json", "--skip-git-repo-check", "-s", "workspace-write", ...(model ? ["-m", model] : []), prompt],
|
|
2077
|
+
parse(o, line) {
|
|
2078
|
+
const ev: CliEvent = {}
|
|
2079
|
+
if (typeof o.type === "string") ev.status = o.type
|
|
2080
|
+
if (o.type === "thread.started") ev.session = str(o.thread_id)
|
|
2081
|
+
else if (o.type === "item.completed") {
|
|
2082
|
+
const item = o.item as any
|
|
2083
|
+
if (item?.type === "agent_message" && typeof item.text === "string") ev.text = item.text
|
|
2084
|
+
} else if (o.type === "error" || o.type === "turn.failed") {
|
|
2085
|
+
ev.error = str(o.message) ?? str((o.error as any)?.message) ?? line
|
|
530
2086
|
}
|
|
531
|
-
|
|
532
|
-
|
|
2087
|
+
return ev
|
|
2088
|
+
},
|
|
2089
|
+
folder(session) {
|
|
2090
|
+
const home = process.env.CODEX_HOME || path.join(os.homedir(), ".codex")
|
|
2091
|
+
return path.join(home, "generated_images", session)
|
|
2092
|
+
},
|
|
2093
|
+
unavailable(error) {
|
|
2094
|
+
const l = error.toLowerCase()
|
|
2095
|
+
return l.includes("usage limit") || l.includes("quota") ? `codex: ${cap(error, 400)}` : undefined
|
|
2096
|
+
},
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
/** Codex CLI (§7G.10): image via `codex exec --json` (ChatGPT plan). */
|
|
2100
|
+
export class CodexMediaClient extends CliMediaClient {
|
|
2101
|
+
constructor(config: MediaClientConfig = {}) {
|
|
2102
|
+
super(CODEX_SPEC, config)
|
|
533
2103
|
}
|
|
2104
|
+
}
|
|
2105
|
+
|
|
2106
|
+
const AGY_SPEC: CliSpec = {
|
|
2107
|
+
name: "agy",
|
|
2108
|
+
command: "agy",
|
|
2109
|
+
templates: {
|
|
2110
|
+
image: "Generate exactly one image with your image generation tool: {prompt}. Do not write code. Reply only with the absolute path where the image was saved.",
|
|
2111
|
+
},
|
|
2112
|
+
argv: (prompt, _cwd, model) => ["-p", prompt, "--output-format", "stream-json", "--dangerously-skip-permissions", ...(model ? ["--model", model] : [])],
|
|
2113
|
+
parse(o) {
|
|
2114
|
+
switch (o.event) {
|
|
2115
|
+
case "init":
|
|
2116
|
+
return { session: str(o.conversation_id), status: "init" }
|
|
2117
|
+
case "step_update": {
|
|
2118
|
+
const su = o.step_update as any
|
|
2119
|
+
const t = str(su?.tool_name)
|
|
2120
|
+
const st = str(su?.state)
|
|
2121
|
+
return t && st ? { status: `tool ${t} ${st}` } : {}
|
|
2122
|
+
}
|
|
2123
|
+
case "result": {
|
|
2124
|
+
const r = o.result as any
|
|
2125
|
+
const ev: CliEvent = { session: str(r?.conversation_id), text: str(r?.response), status: `result ${str(r?.status) ?? ""}` }
|
|
2126
|
+
const low = (ev.text ?? "").toLowerCase()
|
|
2127
|
+
if (r?.status === "SUCCESS" && (low.includes("quota") || low.includes("resource_exhausted"))) {
|
|
2128
|
+
// Live: agy reports an exhausted image quota in a SUCCESS result's text.
|
|
2129
|
+
ev.error = ev.text
|
|
2130
|
+
}
|
|
2131
|
+
if (r?.status !== "SUCCESS") {
|
|
2132
|
+
ev.error = (str(r?.error) || undefined) ?? (str(r?.error?.message) || undefined) ?? (ev.text || undefined)
|
|
2133
|
+
?? JSON.stringify(sortKeys(r ?? null))
|
|
2134
|
+
}
|
|
2135
|
+
const usage = reportedUsage({ inputTokens: r?.usage?.input_tokens, outputTokens: r?.usage?.output_tokens })
|
|
2136
|
+
if (usage) ev.usage = usage
|
|
2137
|
+
return ev
|
|
2138
|
+
}
|
|
2139
|
+
case "error":
|
|
2140
|
+
return { error: str(o.message) }
|
|
2141
|
+
default:
|
|
2142
|
+
return {}
|
|
2143
|
+
}
|
|
2144
|
+
},
|
|
2145
|
+
folder: (session) => path.join(os.homedir(), ".gemini", "antigravity-cli", "brain", session),
|
|
2146
|
+
unavailable(error) {
|
|
2147
|
+
const l = error.toLowerCase()
|
|
2148
|
+
return l.includes("quota") || l.includes("usage limit") ? `agy: ${cap(error, 400)}` : undefined
|
|
2149
|
+
},
|
|
2150
|
+
}
|
|
534
2151
|
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
2152
|
+
// ---- createMedia: routing + tools (§7G.3, §7G.6) ----------------------------------------------
|
|
2153
|
+
|
|
2154
|
+
type VerbInput<T> = T & MediaCallOptions
|
|
2155
|
+
export type ImageArgs = { prompt: string; width?: number; height?: number; aspectRatio?: string; seed?: number }
|
|
2156
|
+
export type VideoArgs = { prompt: string; image?: string; seconds?: number; aspectRatio?: string }
|
|
2157
|
+
export type SpeechArgs = { text: string; voice?: string }
|
|
2158
|
+
export type MusicArgs = { prompt: string; seconds?: number; instrumental?: boolean; lyrics?: string }
|
|
2159
|
+
export type TranscribeArgs = { audio: string; language?: string }
|
|
2160
|
+
export type CloneVoiceArgs = { name: string; samples: string[] }
|
|
2161
|
+
export type TalkArgs = { image: string; audio?: string; text?: string; voice?: string }
|
|
2162
|
+
|
|
2163
|
+
function acceptedArgs(client: MediaClient, verb: string): MediaArgSpec[] {
|
|
2164
|
+
if (typeof client.args === "function") return client.args(verb)
|
|
2165
|
+
return (VERB_SPEC.get(verb)?.args ?? []).map((a) => ({ name: a.name, required: a.required === true }))
|
|
2166
|
+
}
|
|
2167
|
+
|
|
2168
|
+
const isUnavailable = (e: unknown) => e instanceof MediaUnavailableError || (e instanceof MediaError && e.status === "unavailable")
|
|
2169
|
+
|
|
2170
|
+
/** Code-point order (plain `<` compares UTF-16 units, which misorders astral characters). */
|
|
2171
|
+
function byCodePoint(a: string, b: string): number {
|
|
2172
|
+
const x = Array.from(a), y = Array.from(b)
|
|
2173
|
+
for (let i = 0; i < Math.min(x.length, y.length); i++) {
|
|
2174
|
+
const d = x[i].codePointAt(0)! - y[i].codePointAt(0)!
|
|
2175
|
+
if (d) return d
|
|
538
2176
|
}
|
|
2177
|
+
return x.length - y.length
|
|
2178
|
+
}
|
|
539
2179
|
|
|
540
|
-
|
|
541
|
-
|
|
2180
|
+
/** D15: object keys sorted by code point at every depth; array order kept. */
|
|
2181
|
+
function sortKeys(v: unknown): unknown {
|
|
2182
|
+
if (Array.isArray(v)) return v.map(sortKeys)
|
|
2183
|
+
if (v && typeof v === "object") {
|
|
2184
|
+
const keys = Object.keys(v).sort(byCodePoint)
|
|
2185
|
+
return Object.fromEntries(keys.map((k) => [k, sortKeys((v as any)[k])]))
|
|
542
2186
|
}
|
|
543
|
-
|
|
544
|
-
|
|
2187
|
+
return v
|
|
2188
|
+
}
|
|
2189
|
+
|
|
2190
|
+
/** The seven verbs over an ordered list of clients; falls through only on `unavailable`. */
|
|
2191
|
+
export class Media {
|
|
2192
|
+
readonly clients: MediaClient[]
|
|
2193
|
+
|
|
2194
|
+
constructor(clients: MediaClient | MediaClient[]) {
|
|
2195
|
+
const list = Array.isArray(clients) ? clients : [clients]
|
|
2196
|
+
if (list.length === 0) throw new MediaError("media: no clients")
|
|
2197
|
+
this.clients = list
|
|
545
2198
|
}
|
|
546
|
-
|
|
547
|
-
|
|
2199
|
+
|
|
2200
|
+
supports(verb: MediaVerb | string): boolean {
|
|
2201
|
+
return this.clients.some((c) => c.supports(verb))
|
|
548
2202
|
}
|
|
549
|
-
|
|
550
|
-
|
|
2203
|
+
|
|
2204
|
+
/** Every verb some source supports, in canonical verb order. */
|
|
2205
|
+
verbs(): MediaVerb[] {
|
|
2206
|
+
return MEDIA_VERBS.filter((v) => this.supports(v.verb)).map((v) => v.verb)
|
|
551
2207
|
}
|
|
552
|
-
|
|
553
|
-
|
|
2208
|
+
|
|
2209
|
+
/** "style" when every source is a loaded style, else "client". */
|
|
2210
|
+
private noun(): string {
|
|
2211
|
+
return this.clients.every((c) => c instanceof LoadedMediaStyle) ? "style" : "client"
|
|
554
2212
|
}
|
|
555
|
-
|
|
556
|
-
|
|
2213
|
+
|
|
2214
|
+
/** Generic entry point: route `verb` with canonical `args`. */
|
|
2215
|
+
async generate(verb: MediaVerb | string, args: Record<string, unknown> = {}, ctx: MediaContext = {}): Promise<MediaResult> {
|
|
2216
|
+
return (await this.route(verb, args, ctx)).result
|
|
557
2217
|
}
|
|
558
2218
|
|
|
559
|
-
|
|
2219
|
+
private async route(verb: string, rawArgs: Record<string, unknown>, ctx: MediaContext): Promise<{ result: MediaResult; client: string }> {
|
|
2220
|
+
const candidates = this.clients.filter((c) => c.supports(verb))
|
|
2221
|
+
if (candidates.length === 0) throw new MediaError(`media: no ${this.noun()} supports "${verb}"`)
|
|
2222
|
+
const args = present(rawArgs)
|
|
2223
|
+
const canon = (VERB_SPEC.get(verb)?.args ?? []).map((a) => a.name)
|
|
2224
|
+
// D7/D19: a key no client can ever take fails the call at once.
|
|
2225
|
+
for (const key of Object.keys(args).sort(byCodePoint)) {
|
|
2226
|
+
if (!canon.includes(key)) throw new MediaError(`${verb}: argument "${key}" not supported`)
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2229
|
+
const messages: string[] = []
|
|
2230
|
+
for (const c of candidates) {
|
|
2231
|
+
// D19: a candidate that cannot take these args is skipped like `unavailable` — first included.
|
|
2232
|
+
const acc = acceptedArgs(c, verb)
|
|
2233
|
+
const ok = new Set(acc.map((a) => a.name))
|
|
2234
|
+
const bad = Object.keys(args).sort(byCodePoint).find((k) => !ok.has(k))
|
|
2235
|
+
if (bad !== undefined) {
|
|
2236
|
+
messages.push(`${c.name}: argument "${bad}" not supported`)
|
|
2237
|
+
continue
|
|
2238
|
+
}
|
|
2239
|
+
const requiredNames = acc.filter((a) => a.required).map((a) => a.name)
|
|
2240
|
+
const order = [...canon.filter((n) => requiredNames.includes(n)), ...requiredNames.filter((n) => !canon.includes(n))]
|
|
2241
|
+
const miss = order.find((key) => !(key in args))
|
|
2242
|
+
if (miss !== undefined) {
|
|
2243
|
+
messages.push(`${c.name}: missing required argument "${miss}"`)
|
|
2244
|
+
continue
|
|
2245
|
+
}
|
|
2246
|
+
const callArgs = args
|
|
2247
|
+
const fn = (c as any)[verb] as ((a: Record<string, unknown>, x: MediaContext) => Promise<MediaResult>) | undefined
|
|
2248
|
+
if (typeof fn !== "function") throw new MediaError(`media: client "${c.name}" has no ${verb} method`)
|
|
2249
|
+
try {
|
|
2250
|
+
const result = await fn.call(c, callArgs, ctx)
|
|
2251
|
+
if (result.style === undefined) result.style = c instanceof LoadedMediaStyle ? c.id : c.name
|
|
2252
|
+
if (result.model === undefined && c instanceof LoadedMediaStyle) {
|
|
2253
|
+
const model = c.config.models?.[verb as MediaVerb] || c.config.model
|
|
2254
|
+
if (model) result.model = model
|
|
2255
|
+
}
|
|
2256
|
+
return { result, client: c.name }
|
|
2257
|
+
} catch (e) {
|
|
2258
|
+
if (!isUnavailable(e)) throw e
|
|
2259
|
+
messages.push((e as Error).message)
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
throw new MediaUnavailableError(`media: no client available for "${verb}": ${messages.join("; ")}`)
|
|
2263
|
+
}
|
|
2264
|
+
|
|
2265
|
+
private typed(verb: MediaVerb, input: object): Promise<MediaResult> {
|
|
2266
|
+
const { providerOptions, signal, deadline, progress, ...args } = input as Record<string, any>
|
|
2267
|
+
const ctx: MediaContext = {}
|
|
2268
|
+
if (providerOptions !== undefined) ctx.providerOptions = providerOptions
|
|
2269
|
+
if (signal !== undefined) ctx.signal = signal
|
|
2270
|
+
if (deadline !== undefined) ctx.deadline = deadline
|
|
2271
|
+
if (progress !== undefined) ctx.progress = progress
|
|
2272
|
+
return this.generate(verb, args, ctx)
|
|
2273
|
+
}
|
|
2274
|
+
|
|
2275
|
+
image(input: VerbInput<ImageArgs>) { return this.typed("image", input) }
|
|
2276
|
+
video(input: VerbInput<VideoArgs>) { return this.typed("video", input) }
|
|
2277
|
+
speech(input: VerbInput<SpeechArgs>) { return this.typed("speech", input) }
|
|
2278
|
+
music(input: VerbInput<MusicArgs>) { return this.typed("music", input) }
|
|
2279
|
+
transcribe(input: VerbInput<TranscribeArgs>) { return this.typed("transcribe", input) }
|
|
2280
|
+
cloneVoice(input: VerbInput<CloneVoiceArgs>) { return this.typed("cloneVoice", input) }
|
|
2281
|
+
talk(input: VerbInput<TalkArgs>) { return this.typed("talk", input) }
|
|
2282
|
+
|
|
2283
|
+
/** One Tool (source "media") per verb any client supports, in `verbs.json` order (§7G.6). */
|
|
560
2284
|
tools(): Tool[] {
|
|
561
2285
|
const out: Tool[] = []
|
|
562
2286
|
for (const v of MEDIA_VERBS) {
|
|
563
|
-
const
|
|
564
|
-
if (!
|
|
565
|
-
const
|
|
2287
|
+
const first = this.clients.find((c) => c.supports(v.verb))
|
|
2288
|
+
if (!first) continue
|
|
2289
|
+
const acc = new Map(acceptedArgs(first, v.verb).map((a) => [a.name, a]))
|
|
2290
|
+
const mapped = v.args.filter((a) => acc.has(a.name))
|
|
566
2291
|
const properties: Record<string, unknown> = {}
|
|
567
2292
|
for (const a of mapped) {
|
|
568
2293
|
properties[a.name] = a.type === "array"
|
|
569
2294
|
? { type: "array", items: { type: a.items }, description: a.description }
|
|
570
2295
|
: { type: a.type, description: a.description }
|
|
571
2296
|
}
|
|
572
|
-
const inputSchema: JSONSchema = { type: "object", properties, required: mapped.filter((a) => a.required).map((a) => a.name) }
|
|
573
|
-
const
|
|
574
|
-
const mode = resolved.spec.mode
|
|
2297
|
+
const inputSchema: JSONSchema = { type: "object", properties, required: mapped.filter((a) => acc.get(a.name)!.required).map((a) => a.name) }
|
|
2298
|
+
const verb = v.verb
|
|
575
2299
|
out.push({
|
|
576
2300
|
name: v.tool,
|
|
577
2301
|
description: v.description,
|
|
578
2302
|
inputSchema,
|
|
579
2303
|
source: "media",
|
|
580
|
-
// Each call is an independent remote job: always concurrency-safe (§7G, §8 Tool scheduling).
|
|
581
2304
|
concurrencySafe: () => true,
|
|
582
2305
|
execute: async (args: Record<string, unknown>, ctx?: ToolContext): Promise<ToolResult> => {
|
|
583
2306
|
try {
|
|
584
|
-
const r = await this.
|
|
585
|
-
const metadata: Record<string, unknown> = { source: "media", verb
|
|
586
|
-
if (
|
|
2307
|
+
const { result: r, client } = await this.route(verb, args ?? {}, { signal: ctx?.signal })
|
|
2308
|
+
const metadata: Record<string, unknown> = { source: "media", verb, client }
|
|
2309
|
+
if (r.jobId !== undefined) metadata.jobId = r.jobId
|
|
587
2310
|
metadata.status = r.status
|
|
588
|
-
if (
|
|
589
|
-
|
|
590
|
-
return { isError: false, output:
|
|
2311
|
+
if (r.usage !== undefined) metadata.usage = r.usage
|
|
2312
|
+
if (VERB_KIND[verb] === "text") return { isError: false, output: r.text ?? "", metadata }
|
|
2313
|
+
return { isError: false, output: toolOutput(verb, r.files), parts: r.files.map(filePart), metadata }
|
|
591
2314
|
} catch (e) {
|
|
592
2315
|
return { isError: true, output: e instanceof Error ? e.message : String(e) }
|
|
593
2316
|
}
|
|
@@ -598,7 +2321,47 @@ export class Media {
|
|
|
598
2321
|
}
|
|
599
2322
|
}
|
|
600
2323
|
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
2324
|
+
const WORD: Record<MediaFile["type"], string> = { image: "image", file: "video", audio: "audio" }
|
|
2325
|
+
|
|
2326
|
+
function toolOutput(verb: MediaVerb, files: MediaFile[]): string {
|
|
2327
|
+
const word = files.length ? WORD[files[0].type] : VERB_KIND[verb] === "audio" ? "audio" : VERB_KIND[verb]
|
|
2328
|
+
const lines = files.map((f) => f.path ?? f.url ?? `inline ${f.mimeType} ${f.data ? Buffer.from(f.data, "base64").byteLength : 0} bytes`)
|
|
2329
|
+
return `${word} ready: ` + lines.join("\n")
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2332
|
+
/** §7G.6: a part never holds a path — local files are read now as data. */
|
|
2333
|
+
function filePart(f: MediaFile): ContentPart {
|
|
2334
|
+
const part = partOf(f)
|
|
2335
|
+
if (f.path !== undefined) PART_PATH.set(part, f.path)
|
|
2336
|
+
return part
|
|
2337
|
+
}
|
|
2338
|
+
|
|
2339
|
+
function partOf(f: MediaFile): ContentPart {
|
|
2340
|
+
if (f.data !== undefined) return { type: f.type, mimeType: f.mimeType, data: f.data }
|
|
2341
|
+
if (f.url !== undefined && isHttp(f.url)) return { type: f.type, mimeType: f.mimeType, url: f.url }
|
|
2342
|
+
const local = f.path ?? (f.url?.startsWith("file://") ? f.url.slice("file://".length) : undefined)
|
|
2343
|
+
return { type: f.type, mimeType: f.mimeType, data: local !== undefined ? fs.readFileSync(local).toString("base64") : "" }
|
|
2344
|
+
}
|
|
2345
|
+
|
|
2346
|
+
/** §7G.12 D20: the local path behind a media tool's part, kept off the part itself (§1B). */
|
|
2347
|
+
const PART_PATH = new WeakMap<ContentPart, string>()
|
|
2348
|
+
|
|
2349
|
+
/** The saved file's absolute path for a part a media tool produced, if it has one (D20). */
|
|
2350
|
+
export function mediaPartPath(part: ContentPart): string | undefined {
|
|
2351
|
+
return PART_PATH.get(part)
|
|
2352
|
+
}
|
|
2353
|
+
|
|
2354
|
+
/** Route the seven verbs over one client or an ordered list (§7G.3). */
|
|
2355
|
+
export function createMedia(clients: MediaClient | MediaClient[]): Media {
|
|
2356
|
+
return new Media(clients)
|
|
2357
|
+
}
|
|
2358
|
+
|
|
2359
|
+
/** One-shot sugar: `createMedia(loadMediaStyle(style, config)).generate(verb, args)`. */
|
|
2360
|
+
export function generateMedia(
|
|
2361
|
+
style: MediaStylePreset | string,
|
|
2362
|
+
verb: MediaVerb | string,
|
|
2363
|
+
args: Record<string, unknown> = {},
|
|
2364
|
+
config: MediaClientConfig = {},
|
|
2365
|
+
): Promise<MediaResult> {
|
|
2366
|
+
return createMedia(loadMediaStyle(style, config)).generate(verb, args)
|
|
604
2367
|
}
|