toolnexus 0.23.0 → 0.24.0

package/README.md

@@ -135,6 +135,29 @@ const out = await gate(c, state, questions, [
   filter, content and model batteries into `hooks`. A `beforeLLM` hook may return `{ model }` to
   send another model for that turn only.
 
+## Media generation
+
+Generate images, video and speech (plus transcription, voice cloning and talking-head video)
+from a **JSON style** instead of provider code. Async jobs are started and polled for you. The same
+verbs become agent tools (`generate_image`, `generate_video`, `generate_speech`, …), and whatever
+they produce lands in `RunResult.files`. `examples/media-styles/a2e.json` ships first and reads
+`A2E_API_KEY` at call time. Its result URLs expire in 3–30 days, so save what you keep. SPEC §7G;
+guide: <https://muthuishere.github.io/toolnexus/media/>.
+
+```ts
+import { loadMediaStyle, createMedia, createToolkit } from "toolnexus"
+
+const media = createMedia(loadMediaStyle("examples/media-styles/a2e.json"), {
+  onProgress: ({ verb, status }) => console.log(verb, status),
+})
+const img = await media.image({ prompt: "a red fox in fresh snow", width: 1024, height: 768 })
+const vid = await media.video({ prompt: "the fox blinks", image: img.files[0].url, seconds: 5 })
+
+const tk = await createToolkit({ media })            // the agent gets generate_image, …
+const res = await agent.run("Draw a fox, then animate it.", { toolkit: tk })
+for (const f of res.files) console.log(f.from, f.mimeType, f.url)
+```
+
 ## Documentation
 
 Everything else — the full surface, with runnable examples — lives on the docs site:

package/dist/agents/loop.js

@@ -76,6 +76,7 @@ export async function runGated(ask, prompt, completion) {
         throw new Error("toolnexus: completion.verify is required");
     }
     let accumulated = [];
+    let accFiles = [];
     let last;
     let reason = "";
     for (let attempt = 1; attempt <= maxAttempts; attempt++) {
@@ -84,7 +85,9 @@ export async function runGated(ask, prompt, completion) {
         // The gate judges the ACCUMULATED work, so an agent cannot escape it by
         // declining to re-declare its plan on a retry.
         accumulated = [...accumulated, ...r.toolCalls];
-        const judged = { ...r, toolCalls: accumulated };
+        // RunResult.files is derived from toolCalls, so it accumulates with them (§8).
+        accFiles = [...accFiles, ...(r.files ?? [])];
+        const judged = { ...r, toolCalls: accumulated, files: accFiles };
         last = judged;
         if (r.status && r.status !== "done") {
             // The run stopped for its own reason (suspension, budget). If the gate was

package/dist/agents/runtime.js

@@ -751,7 +751,7 @@ export class AgentRuntime {
         const team = [...(parent.def.team ?? [])].sort();
         const advertised = team.map((n) => (this.registry[n] ? `${n}: ${this.registry[n].does}` : n)).join("; ");
         const rt = this;
-        return defineTool({
+        const tool = defineTool({
             name: "task",
             description: `Delegate a subtask to an isolated subagent (it runs on a fresh transcript and returns only its final answer). Available agents — ${advertised}`,
             inputSchema: {
@@ -813,5 +813,8 @@ export class AgentRuntime {
                 };
             },
         });
+        // Each subagent is isolated and the runtime's own gate schedules the children,
+        // so a fan-out of task calls runs concurrently (SPEC §8 "Tool scheduling").
+        return { ...tool, concurrencySafe: () => true };
     }
 }

package/dist/client.d.ts

@@ -7,7 +7,7 @@ import type { Toolkit } from "./toolkit.js";
 import type { ToolHints, ToolResult, Request, Answer } from "./types.js";
 import { type PromptCacheOptions } from "./promptcache.js";
 import { type ToolTier, type ToolTierOptions } from "./tooltiers.js";
-import type { PromptInput, UnsupportedPartMode } from "./content.js";
+import type { ContentPart, PromptInput, UnsupportedPartMode } from "./content.js";
 import type { TranslateRequest, TranslateResult } from "./translate.js";
 export type ClientStyle = "openai" | "anthropic";
 export interface ClientOptions {
@@ -26,6 +26,10 @@ export interface ClientOptions {
     headers?: Record<string, string>;
     systemPrompt?: string;
     maxTurns?: number;
+    /** Cap on concurrency-safe tool calls of one response running at once (SPEC §8 "Tool
+     * scheduling"). Absent ⇒ env `TOOLNEXUS_MAX_TOOL_CONCURRENCY` when a positive integer, else 10.
+     * Unsafe calls (anything not read-only) always run alone. */
+    maxToolConcurrency?: number;
     hooks?: Hooks;
     /** Retries on transient LLM errors (`429`/`500`/`502`/`503`/`504`/`529` + network). Default 2.
      * Widen the status set with `retryableStatuses`. */
@@ -299,7 +303,19 @@ export interface ToolCallRecord {
     output: string;
     isError: boolean;
     metadata?: Record<string, unknown>;
+    /** §8: the ToolResult's parts — present iff the result had parts. */
+    parts?: ContentPart[];
 }
+/** §8 RunResult.files entry: a non-text part a tool call produced this run. */
+export type RunFile = {
+    type: "image" | "file" | "audio";
+    mimeType: string;
+    url?: string;
+    data?: string;
+    name?: string;
+    from: string;
+    callId: string;
+};
 /** Token usage, summed across every LLM round trip in the run. */
 export interface Usage {
     promptTokens: number;
@@ -319,6 +335,8 @@ export interface RunResult {
     messages: any[];
     /** Every tool call made, with its output, error flag, and metadata. */
     toolCalls: ToolCallRecord[];
+    /** Every non-text part produced by tool calls this run, in call order (§8). Always present. */
+    files: RunFile[];
     /** Total number of tool calls (= toolCalls.length). */
     toolCallCount: number;
     /** Number of LLM round trips. */
@@ -491,6 +509,10 @@ export declare class Client {
     /** Run one tool through beforeTool/afterTool hooks (mutate args, short-circuit, transform result). */
     private runTool;
     private runToolDirect;
+    /** Runs one response's calls under the tool schedule (SPEC §8 "Tool scheduling"). */
+    private schedule;
+    /** One call's safety; looks through `call_tool` to the tool it names. Unknown ⇒ unsafe. */
+    private callSafe;
     /** ToolTiers `call_tool {name, arguments}` -> the real call; undefined when tiers are off. */
     private unwrapCall;
     /** The RunResult.toolCalls entry: a `call_tool` call is recorded as the real tool with its

package/dist/client.js

@@ -3,6 +3,7 @@ import { toOpenAI, toAnthropic } from "./adapters.js";
 import { applyPromptCache } from "./promptcache.js";
 import { buildToolPlan, toolSearch, unwrapCallTool } from "./tooltiers.js";
 import { isRetryableStatus, retryAfterMs } from "./retry.js";
+import { isConcurrencySafe, resolveMaxToolConcurrency, scheduleCalls } from "./toolconcurrency.js";
 import { toOpenAIWire, toAnthropicWire, checkPromptParts } from "./wire.js";
 import { finishReasonFor, argsString, openAIMessagesToAnthropic, openAIToolsToAnthropic, openAIToolChoiceToAnthropic, hasSystemMessage, } from "./translate.js";
 /**
@@ -106,6 +107,20 @@ function delay(ms, signal) {
         signal.addEventListener("abort", () => { clearTimeout(t); reject(signal.reason ?? new Error("aborted")); }, { once: true });
     });
 }
+/** Model tool-call id of each record, kept off the record itself (§8 shape has no id). */
+const callIds = new WeakMap();
+/** RunResult.files: every non-text part of every tool call, in call order (§8). */
+function runFiles(toolCalls) {
+    const out = [];
+    for (const r of toolCalls) {
+        for (const p of r.parts ?? []) {
+            if (p.type === "text")
+                continue;
+            out.push({ ...p, from: r.name, callId: callIds.get(r) ?? "" });
+        }
+    }
+    return out;
+}
 function addUsage(acc, raw, style) {
     if (!raw)
         return;
@@ -718,15 +733,34 @@ export class Client {
         this.emitTool(name, source, result, t0);
         return { args: a, result };
     }
+    /** Runs one response's calls under the tool schedule (SPEC §8 "Tool scheduling"). */
+    schedule(toolkit, items, key, fn) {
+        return scheduleCalls(items, (item) => {
+            const [name, args] = key(item);
+            return this.callSafe(toolkit, name, args);
+        }, resolveMaxToolConcurrency(this.opts.maxToolConcurrency), fn);
+    }
+    /** One call's safety; looks through `call_tool` to the tool it names. Unknown ⇒ unsafe. */
+    callSafe(toolkit, name, args) {
+        if (this.opts.toolTiers && name === "tool_search")
+            return true;
+        const inner = this.unwrapCall(name, args ?? {});
+        const tool = toolkit?.get(inner ? inner.name : name);
+        return tool ? isConcurrencySafe(tool, inner ? inner.args : (args ?? {})) : false;
+    }
     /** ToolTiers `call_tool {name, arguments}` -> the real call; undefined when tiers are off. */
     unwrapCall(name, args) {
         return this.opts.toolTiers ? unwrapCallTool(name, args) : undefined;
     }
     /** The RunResult.toolCalls entry: a `call_tool` call is recorded as the real tool with its
      *  inner arguments, the name hooks and stream events see (the transcript keeps the wrapper). */
-    toolCallRecord(name, args, result) {
+    toolCallRecord(name, args, result, callId) {
         const inner = this.unwrapCall(name, args);
-        return { name: inner ? inner.name : name, args: inner ? inner.args : args, output: result.output, isError: result.isError, metadata: result.metadata };
+        const rec = { name: inner ? inner.name : name, args: inner ? inner.args : args, output: result.output, isError: result.isError, metadata: result.metadata };
+        if (result.parts?.length)
+            rec.parts = result.parts;
+        callIds.set(rec, callId ?? "");
+        return rec;
     }
     /** The stream's tool_call event, naming the real tool for a `call_tool` call. */
     callEvent(id, name, args) {
@@ -886,7 +920,7 @@ export class Client {
                     return this.endRun(runStart, lastModel, msg.content ?? "", messages, toolCalls, turns, usage);
                 // execute all tool calls in this turn concurrently (true parallel tool calling)
                 done.clear();
-                const settled = await untilAborted(signal, Promise.all(calls.map(async (call) => {
+                const settled = await untilAborted(signal, this.schedule(toolkit, calls, (call) => [call.function.name, safeJson(call.function.arguments)], async (call) => {
                     let { args, result } = await this.runTool(toolkit, call.function.name, safeJson(call.function.arguments), call.id, turn, signal);
                     let halted;
                     const req = pendingOf(result);
@@ -898,12 +932,12 @@ export class Client {
                     if (!halted && call.id)
                         done.set(call.id, result);
                     return { call, args, result, halted };
-                })));
+                }));
                 // Record in tool-call order; on the FIRST durable halt, surface it and stop — deterministic,
                 // and the later suspensions' placeholder results never enter the transcript (they re-suspend
                 // on resume). Mirrors the streaming path.
                 for (const s of settled) {
-                    toolCalls.push(this.toolCallRecord(s.call.function.name, s.args, s.result));
+                    toolCalls.push(this.toolCallRecord(s.call.function.name, s.args, s.result, s.call.id));
                     messages.push(toolMessage(s.call.id, s.call.function.name, s.result));
                     if (s.halted)
                         return this.pendingRun(runStart, lastModel, s.halted, messages, toolCalls, turns, usage);
@@ -919,7 +953,7 @@ export class Client {
     }
     result(model, text, messages, toolCalls, turns, usage, exhausted = false) {
         const incomplete = exhausted && text === "";
-        const r = { text, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model, status: incomplete ? "incomplete" : "done" };
+        const r = { text, messages, toolCalls, files: runFiles(toolCalls), toolCallCount: toolCalls.length, turns, usage, model, status: incomplete ? "incomplete" : "done" };
         if (incomplete)
             r.limit = "maxTurns";
         return r;
@@ -927,7 +961,7 @@ export class Client {
     /** §10: a run halted because a tool suspended and no `waitFor` was configured. */
     pendingRun(runStart, model, request, messages, toolCalls, turns, usage) {
         this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart });
-        return { text: request.prompt, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model, status: "pending", pending: request };
+        return { text: request.prompt, messages, toolCalls, files: runFiles(toolCalls), toolCallCount: toolCalls.length, turns, usage, model, status: "pending", pending: request };
     }
     /**
      * §10: resolve a suspended tool result. Calls `waitFor(request)`, and on `ok` re-executes the
@@ -1008,7 +1042,7 @@ export class Client {
                 }
                 // execute all tool_use blocks in this turn concurrently (true parallel tool calling)
                 done.clear();
-                const settled = await untilAborted(signal, Promise.all(uses.map(async (use) => {
+                const settled = await untilAborted(signal, this.schedule(toolkit, uses, (use) => [use.name, use.input ?? {}], async (use) => {
                     let { args, result } = await this.runTool(toolkit, use.name, use.input ?? {}, use.id, turn, signal);
                     let halted;
                     const req = pendingOf(result);
@@ -1020,14 +1054,14 @@ export class Client {
                     if (!halted && use.id)
                         done.set(use.id, result);
                     return { use, args, result, halted };
-                })));
+                }));
                 // Record in tool-call order; on the FIRST durable halt, include that tool_result block, stop
                 // building the content, and surface it — deterministic (G3), later suspensions re-suspend on
                 // resume. Mirrors the OpenAI path and the streaming loops.
                 const content = [];
                 let halted;
                 for (const s of settled) {
-                    toolCalls.push(this.toolCallRecord(s.use.name, s.args, s.result));
+                    toolCalls.push(this.toolCallRecord(s.use.name, s.args, s.result, s.use.id));
                     content.push(toolResultBlock(s.use.id, s.result));
                     if (s.halted) {
                         halted = s.halted;
@@ -1136,12 +1170,12 @@ export class Client {
                 for (const c of calls)
                     yield this.callEvent(c.id, c.name, safeJson(c.args));
                 done.clear();
-                const settled = await untilAborted(signal, Promise.all(calls.map(async (c) => {
+                const settled = await untilAborted(signal, this.schedule(toolkit, calls, (c) => [c.name, safeJson(c.args)], async (c) => {
                     const r = await this.runTool(toolkit, c.name, safeJson(c.args), c.id, turn, signal);
                     if (!pendingOf(r.result) && c.id)
                         done.set(c.id, r.result);
                     return r;
-                })));
+                }));
                 for (let i = 0; i < calls.length; i++) {
                     const c = calls[i];
                     let { args, result } = settled[i];
@@ -1151,13 +1185,13 @@ export class Client {
                         const r = await this.resolvePending(toolkit, c.name, args, req, c.id, turn, signal);
                         result = r.result;
                         if (r.halted) {
-                            toolCalls.push(this.toolCallRecord(c.name, args, result));
+                            toolCalls.push(this.toolCallRecord(c.name, args, result, c.id));
                             messages.push(toolMessage(c.id, c.name, result));
                             yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) };
                             return;
                         }
                     }
-                    toolCalls.push(this.toolCallRecord(c.name, args, result));
+                    toolCalls.push(this.toolCallRecord(c.name, args, result, c.id));
                     messages.push(toolMessage(c.id, c.name, result));
                     yield this.resultEvent(c.id, c.name, safeJson(c.args), result);
                 }
@@ -1256,12 +1290,12 @@ export class Client {
                 for (const u of uses)
                     yield this.callEvent(u.id, u.name, u.input);
                 done.clear();
-                const settled = await untilAborted(signal, Promise.all(uses.map(async (u) => {
+                const settled = await untilAborted(signal, this.schedule(toolkit, uses, (u) => [u.name, u.input ?? {}], async (u) => {
                     const r = await this.runTool(toolkit, u.name, u.input ?? {}, u.id, turn, signal);
                     if (!pendingOf(r.result) && u.id)
                         done.set(u.id, r.result);
                     return r;
-                })));
+                }));
                 const results = [];
                 for (let i = 0; i < uses.length; i++) {
                     const u = uses[i];
@@ -1272,14 +1306,14 @@ export class Client {
                         const r = await this.resolvePending(toolkit, u.name, args, req, u.id, turn, signal);
                         result = r.result;
                         if (r.halted) {
-                            toolCalls.push(this.toolCallRecord(u.name, args, result));
+                            toolCalls.push(this.toolCallRecord(u.name, args, result, u.id));
                             results.push(toolResultBlock(u.id, result));
                             messages.push({ role: "user", content: results });
                             yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) };
                             return;
                         }
                     }
-                    toolCalls.push(this.toolCallRecord(u.name, args, result));
+                    toolCalls.push(this.toolCallRecord(u.name, args, result, u.id));
                     results.push(toolResultBlock(u.id, result));
                     yield this.resultEvent(u.id, u.name, u.input, result);
                 }

package/dist/index.d.ts

@@ -22,5 +22,7 @@ export * from "./batteries.js";
 export { State, context, questionMap, ask, gate, decide, applyGate, applyPolicy, answersOf, staticClassifier, Tape, DEFAULT_BANDS, type NamedQuestion, type Band, type Bands, type Answers, type Rule, type GateOutcome, type Policy, } from "./judge.js";
 export * from "./promptcache.js";
 export * from "./tooltiers.js";
+export * from "./toolconcurrency.js";
 export * from "./models.js";
 export * from "./builtin_extras.js";
+export * from "./media.js";

package/dist/index.js

@@ -25,5 +25,7 @@ export * from "./batteries.js";
 export { State, context, questionMap, ask, gate, decide, applyGate, applyPolicy, answersOf, staticClassifier, Tape, DEFAULT_BANDS, } from "./judge.js";
 export * from "./promptcache.js";
 export * from "./tooltiers.js";
+export * from "./toolconcurrency.js";
 export * from "./models.js";
 export * from "./builtin_extras.js";
+export * from "./media.js";

package/dist/media.d.ts

@@ -0,0 +1,126 @@
+import type { Tool } from "./types.js";
+export type MediaVerb = "image" | "video" | "speech" | "transcribe" | "cloneVoice" | "talking";
+export type MediaKind = "image" | "video" | "audio" | "text";
+export interface MediaVerbStyle {
+    mode: "sync" | "async";
+    kind: MediaKind;
+    start: {
+        method?: string;
+        path: string;
+    };
+    defaults?: Record<string, unknown>;
+    args: Record<string, string>;
+    idPath?: string;
+    poll?: {
+        method?: string;
+        path: string;
+        statusPath: string;
+        done?: string[];
+        failed?: string[];
+        errorPath?: string;
+        intervalMs?: number;
+        timeoutMs?: number;
+    };
+    resultPath: string;
+}
+export interface MediaStyle {
+    baseUrl: string;
+    headers?: Record<string, string>;
+    errorEnvelope?: {
+        codePath: string;
+        okValues: unknown[];
+        messagePath?: string;
+    };
+    verbs: Partial<Record<MediaVerb, MediaVerbStyle>>;
+}
+export interface MediaFile {
+    type: "image" | "audio" | "file";
+    mimeType: string;
+    url: string;
+}
+export interface MediaResult {
+    files: MediaFile[];
+    text?: string;
+    jobId?: string;
+    status: string;
+}
+export interface MediaProgress {
+    verb: MediaVerb;
+    jobId: string;
+    status: string;
+}
+export interface MediaOptions {
+    onProgress?: (p: MediaProgress) => void;
+    /** HTTP transport (same seam as the LLM client's `fetch`). Omit ⇒ global fetch. */
+    fetch?: typeof fetch;
+}
+/** Per-call options of the function face. */
+export interface MediaCallOptions {
+    /** Top-level keys overwrite the request body last. Never exposed on the tool face. */
+    providerOptions?: Record<string, unknown>;
+    signal?: AbortSignal;
+}
+export type ImageArgs = {
+    prompt: string;
+    width?: number;
+    height?: number;
+    aspectRatio?: string;
+    seed?: number;
+};
+export type VideoArgs = {
+    prompt: string;
+    image?: string;
+    seconds?: number;
+    aspectRatio?: string;
+};
+export type SpeechArgs = {
+    text: string;
+    voice?: string;
+};
+export type TranscribeArgs = {
+    audio: string;
+};
+export type CloneVoiceArgs = {
+    name: string;
+    samples: string[];
+};
+export type TalkingArgs = {
+    image: string;
+    audio: string;
+};
+export interface MediaVerbArg {
+    name: string;
+    type: string;
+    items?: string;
+    description: string;
+    required?: boolean;
+}
+export interface MediaVerbSpec {
+    verb: MediaVerb;
+    tool: string;
+    description: string;
+    args: MediaVerbArg[];
+}
+/** Canonical verbs — `conformance/media/verbs.json`, embedded verbatim (byte contract). */
+export declare const MEDIA_VERBS: MediaVerbSpec[];
+/** Read one style file (JSON). */
+export declare function loadMediaStyle(path: string): MediaStyle;
+export declare class Media {
+    private readonly opts;
+    private readonly styles;
+    constructor(styles: MediaStyle | MediaStyle[], opts?: MediaOptions);
+    private resolve;
+    /** Generic entry point: run `verb` with normalised `args` (§7G engine). */
+    generate(verb: MediaVerb | string, args: Record<string, unknown>, callOpts?: MediaCallOptions): Promise<MediaResult>;
+    private typed;
+    image(input: ImageArgs & MediaCallOptions): Promise<MediaResult>;
+    video(input: VideoArgs & MediaCallOptions): Promise<MediaResult>;
+    speech(input: SpeechArgs & MediaCallOptions): Promise<MediaResult>;
+    transcribe(input: TranscribeArgs & MediaCallOptions): Promise<MediaResult>;
+    cloneVoice(input: CloneVoiceArgs & MediaCallOptions): Promise<MediaResult>;
+    talking(input: TalkingArgs & MediaCallOptions): Promise<MediaResult>;
+    /** One Tool (source "media") per supported verb, in canonical order (§7G tool face). */
+    tools(): Tool[];
+}
+/** Build a media engine over one style or a list (first style declaring a verb wins). */
+export declare function createMedia(styles: MediaStyle | MediaStyle[], opts?: MediaOptions): Media;