pi-provider-freellmapi 1.0.2 → 1.0.4

package/README.md

@@ -15,6 +15,16 @@ reasoning/vision capabilities.
 pi install npm:pi-provider-freellmapi
 ```
 
+The extension registers three pi providers — all sharing the same model list from the gateway — so you can pick whichever API shape suits your use case:
+
+| Provider | API type | Endpoint |
+|----------|----------|----------|
+| `freellmapi` | `openai-completions` | `/v1/chat/completions` |
+| `freellmapi-responses` | `openai-responses` | `/v1/responses` |
+| `freellmapi-anthropic` | `anthropic-messages` | `/v1/messages` |
+
+Switch provider in the TUI with `/model`, or from the CLI:
+
 ## Configure
 
 Copy the example config to `~/.pi/agent/freellmapi.json` and fill in your key:
@@ -35,7 +45,14 @@ cp freellmapi.example.json ~/.pi/agent/freellmapi.json
   "fetchModels": true,
   "contextWindow": 131072,
   "maxTokens": 16384,
-  "compat": { "supportsDeveloperRole": false }
+  "compat": { "supportsDeveloperRole": false },
+  "endpoints": {
+    "embeddings":     { "enabled": true, "model": "auto" },
+    "images":         { "enabled": true, "model": "auto" },
+    "videos":         { "enabled": true, "model": "inkling" },
+    "speech":         { "enabled": true, "model": "auto" },
+    "transcriptions": { "enabled": true, "model": "whisper-1" }
+  }
 }
 ```
 
@@ -43,18 +60,43 @@ The `apiKey` supports pi's value syntax: a literal, `$ENV_VAR`, `${ENV_VAR}`,
 or `!command`. You can also skip the config file entirely and just export
 `FREELLM_API_KEY`.
 
-- Override the config path with `PI_FREELMAPI_CONFIG=/path/to/config.json`.
+- Override the config path with `PI_FREELLMAPI_CONFIG=/path/to/config.json`.
 - Set `"fetchModels": false` to use only the static `models` list.
 
 ## Usage
 
 ```bash
-pi --provider freellmapi --model auto
-pi --provider freellmapi --model deepseek-v4-flash
+pi --provider freellmapi-responses --model auto
+pi --provider freellmapi-anthropic --model auto
 ```
 
 Or open `/model` in the TUI and pick from the `FreeLLM API` group.
 
+## Endpoint tools
+
+In addition to the chat provider, the extension registers custom tools that
+wrap the gateway's other OpenAI-compatible endpoints. They are enabled by
+default; set `"enabled": false` (`"endpoints": { "images": false }`) or set a
+default `model` per endpoint to customize them.
+
+| Tool | Endpoint | What it does |
+|------|----------|--------------|
+| `freellm_embed` | `POST /embeddings` | Generates embeddings; writes JSON to a file |
+| `freellm_image` | `POST /images/generations` | Generates image(s), returned inline |
+| `freellm_video` | `POST /videos/generations` | Generates a video, polls the task, downloads the file |
+| `freellm_speech` | `POST /audio/speech` | Synthesizes speech, writes audio to a file |
+| `freellm_transcribe` | `POST /audio/transcriptions` | Transcribes a local path or URL, returns text |
+
+Each tool accepts an optional `model` argument. When omitted, the extension
+picks a default from (in priority order): the endpoint's `model` config, model
+ids discovered from `{baseUrl}/models` matching that endpoint, or a
+conventional OpenAI id (e.g. `auto` for embeddings/images/speech, `inkling`, `whisper-1`). If the gateway rejects a default, set the
+endpoint's `model` in config.
+
+Files written by the tools (embeddings JSON, images are returned inline, video,
+speech audio) land in the session working directory with `freellm-*` names, or
+an `outputFile` path you pass to the tool.
+
 ## Config reference
 
 | Field | Default | Description |
@@ -68,6 +110,8 @@ Or open `/model` in the TUI and pick from the `FreeLLM API` group.
 | `maxTokens` | `16384` | Fallback max output tokens |
 | `compat` | `{"supportsDeveloperRole":false}` | Provider compatibility flags |
 | `models` | `[]` | Static models (override discovered models by id) |
+| `headers` | `{}` | Extra headers merged into every tool request |
+| `endpoints` | all enabled | Per-endpoint `enabled` flag and default `model` |
 
 ## License
 

package/freellmapi.example.json

@@ -9,5 +9,34 @@
   "compat": {
     "supportsDeveloperRole": false
   },
-  "models": []
+  "models": [],
+  "headers": {},
+  "endpoints": {
+    "embeddings": {
+      "enabled": true,
+      "model": "auto"
+    },
+    "images": {
+      "enabled": true,
+      "model": "auto"
+    },
+    "videos": {
+      "enabled": true,
+      "model": "inkling"
+    },
+    "speech": {
+      "enabled": true,
+      "model": "auto"
+    },
+    "transcriptions": {
+      "enabled": true,
+      "model": "whisper-1"
+    },
+    "responses": {
+      "enabled": true
+    },
+    "anthropic": {
+      "enabled": true
+    }
+  }
 }

package/index.ts

@@ -1,9 +1,27 @@
 /**
  * pi-provider-freellmapi — register the FreeLLM API gateway as a pi provider.
  *
- * This extension registers `https://freeapi.n.cofire.cn/v1` as an
- * OpenAI-compatible (`openai-completions`) provider and auto-discovers the
- * models your API key can actually use.
+ * This extension registers `https://freeapi.n.cofire.cn/v1` as three
+ * OpenAI-compatible providers and auto-discovers the models your API key can
+ * actually use:
+ *
+ * | Provider | API type | Endpoint |
+ * |----------|----------|----------|
+ * | `freellmapi` | `openai-completions` | `/v1/chat/completions` |
+ * | `freellmapi-responses` | `openai-responses` | `/v1/responses` |
+ * | `freellmapi-anthropic` | `anthropic-messages` | `/v1/messages` |
+ *
+ * It also registers a set of custom tools that wrap the other OpenAI-compatible
+ * endpoints the gateway exposes, so the agent can use them directly:
+ *
+ *   - `POST /embeddings`            → `freellm_embed`
+ *   - `POST /images/generations`    → `freellm_image`
+ *   - `POST /videos/generations`    → `freellm_video` (tasks + polling)
+ *   - `POST /audio/speech`          → `freellm_speech`
+ *   - `POST /audio/transcriptions`  → `freellm_transcribe`
+ *
+ * Each tool is enabled by default and can be turned off (or given a default
+ * model) via the `endpoints` block in the config file.
  *
  * ## Install
  *
@@ -23,7 +41,14 @@
  *   "contextWindow": 131072,                 // fallback context window (tokens)
  *   "maxTokens": 16384,                      // fallback max output tokens
  *   "compat": { "supportsDeveloperRole": false },
- *   "models": []                             // optional static models (override by id)
+ *   "models": [],                            // optional static models (override by id)
+ *   "endpoints": {
+ *     "embeddings":      { "enabled": true, "model": "auto" },
+ *     "images":          { "enabled": true, "model": "auto" },
+ *     "videos":          { "enabled": true, "model": "inkling" },
+ *     "speech":          { "enabled": true, "model": "auto" },
+ *     "transcriptions":  { "enabled": true, "model": "whisper-1" }
+ *   }
  * }
  * ```
  *
@@ -39,9 +64,13 @@
  */
 
 import type { ExtensionAPI, ProviderModelConfig } from "@earendil-works/pi-coding-agent";
-import { readFileSync, existsSync } from "node:fs";
+import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
+import { Type } from "typebox";
+import { execSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { readFile, writeFile, mkdir } from "node:fs/promises";
 import { homedir } from "node:os";
-import { resolve } from "node:path";
+import { basename, dirname, extname, resolve } from "node:path";
 
 // ── Types ──────────────────────────────────────────────────────────────────
 
@@ -53,6 +82,25 @@ interface RemoteModel {
   supported_parameters?: string[];
 }
 
+interface EndpointConfig {
+  /** Whether the corresponding tool is registered (default: true). */
+  enabled?: boolean;
+  /** Default model id used when a tool call omits `model`. */
+  model?: string;
+}
+
+interface EndpointsConfig {
+  embeddings?: EndpointConfig | boolean;
+  images?: EndpointConfig | boolean;
+  videos?: EndpointConfig | boolean;
+  speech?: EndpointConfig | boolean;
+  transcriptions?: EndpointConfig | boolean;
+  /** OpenAI Responses API (`/v1/responses`). */
+  responses?: EndpointConfig | boolean;
+  /** Anthropic Messages API (`/v1/messages`). */
+  anthropic?: EndpointConfig | boolean;
+}
+
 interface ConfigFile {
   name?: string;
   label?: string;
@@ -61,8 +109,20 @@ interface ConfigFile {
   fetchModels?: boolean;
   contextWindow?: number;
   maxTokens?: number;
+  /** Extra request headers merged into every tool request. */
+  headers?: Record<string, string>;
   compat?: Record<string, unknown>;
   models?: ProviderModelConfig[];
+  endpoints?: EndpointsConfig;
+}
+
+type ToolContent = (TextContent | ImageContent)[];
+
+interface HttpInit {
+  method?: string;
+  headers?: Record<string, string>;
+  body?: string | FormData;
+  signal?: AbortSignal;
 }
 
 // ── Defaults ───────────────────────────────────────────────────────────────
@@ -86,6 +146,12 @@ const FALLBACK_MODELS: Array<{ id: string; name: string }> = [
   { id: "claude-sonnet-4-5", name: "Claude Sonnet slot" },
 ];
 
+const VIDEO_TERMINAL_STATUSES = new Set([
+  "completed", "succeeded", "success", "done",
+  "failed", "error", "cancelled", "canceled", "killed", "rejected",
+]);
+const VIDEO_SUCCESS_STATUSES = new Set(["completed", "succeeded", "success", "done"]);
+
 // ── Helpers ────────────────────────────────────────────────────────────────
 
 function loadConfig(configPath: string): ConfigFile {
@@ -100,6 +166,13 @@ function loadConfig(configPath: string): ConfigFile {
   }
 }
 
+/** Normalize `endpoints.<x>` which may be a boolean or an object. */
+function normalizeEndpoint(raw: EndpointConfig | boolean | undefined): EndpointConfig {
+  if (raw === undefined) return {};
+  if (typeof raw === "boolean") return { enabled: raw };
+  return raw;
+}
+
 /**
  * Resolve an apiKey value that may use pi's value syntax:
  *   - `${ENV_VAR}` / `$ENV_VAR` (whole value) → environment variable
@@ -116,6 +189,18 @@ function resolveApiKey(raw: string): string {
   return raw;
 }
 
+/** Resolve an apiKey for a direct tool request, executing `!command` if needed. */
+function resolveApiKeyValue(raw: string): string {
+  if (raw.startsWith("!")) {
+    try {
+      return execSync(raw.slice(1), { encoding: "utf8", timeout: 30000 }).trim();
+    } catch (err) {
+      throw new Error(`[freellmapi] Failed to execute apiKey command: ${raw.slice(1)}`);
+    }
+  }
+  return resolveApiKey(raw);
+}
+
 function isVision(name: string, id: string): boolean {
   return /vision|vl|image|moondream/i.test(`${name} ${id}`);
 }
@@ -173,6 +258,166 @@ function mergeModels(
   return [...map.values()];
 }
 
+/** Pick the first available model id matching any of the patterns, else the fallback. */
+function pickDefaultModel(ids: string[], patterns: RegExp[], fallback: string): string {
+  for (const pattern of patterns) {
+    const match = ids.find((id) => pattern.test(id));
+    if (match) return match;
+  }
+  return fallback;
+}
+
+function guessMimeFromExt(nameOrUrl: string): string | undefined {
+  const ext = extname(nameOrUrl.split("?")[0].split("#")[0]).toLowerCase();
+  const map: Record<string, string> = {
+    ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
+    ".webp": "image/webp", ".gif": "image/gif",
+    ".mp3": "audio/mpeg", ".wav": "audio/wav", ".ogg": "audio/ogg", ".oga": "audio/ogg",
+    ".opus": "audio/opus", ".flac": "audio/flac", ".m4a": "audio/mp4", ".aac": "audio/aac",
+    ".mp4": "video/mp4", ".webm": "video/webm", ".mov": "video/quicktime", ".mpeg": "video/mpeg",
+  };
+  return map[ext];
+}
+
+function guessVideoExt(url: string): string {
+  const ext = extname(url.split("?")[0].split("#")[0]).toLowerCase();
+  return [".mp4", ".webm", ".mov", ".mpeg", ".mkv"].includes(ext) ? ext : ".mp4";
+}
+
+function encodeBase64(bytes: Uint8Array): string {
+  return Buffer.from(bytes).toString("base64");
+}
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+async function readErrorBody(res: Response): Promise<string> {
+  const text = await res.text();
+  try {
+    const json = JSON.parse(text) as { error?: { message?: string }; message?: string };
+    return json?.error?.message ?? json?.message ?? text;
+  } catch {
+    return text;
+  }
+}
+
+/** Unified request helper: auth + config headers + JSON/multipart body handling. */
+async function httpRequest(
+  baseUrl: string,
+  apiKey: string | undefined,
+  extraHeaders: Record<string, string> | undefined,
+  path: string,
+  init: HttpInit = {},
+): Promise<Response> {
+  const reqHeaders = init.headers ?? {};
+  const isForm = typeof (init.body as FormData | undefined)?.append === "function";
+
+  const merged: Record<string, string> = { ...(extraHeaders ?? {}) };
+  for (const [k, v] of Object.entries(reqHeaders)) merged[k] = v;
+
+  const resolvedKey = apiKey ? resolveApiKeyValue(apiKey) : undefined;
+  if (resolvedKey && !("Authorization" in merged) && !("authorization" in merged)) {
+    merged["Authorization"] = `Bearer ${resolvedKey}`;
+  }
+  if (!isForm && !("Content-Type" in merged) && !("content-type" in merged)) {
+    merged["Content-Type"] = "application/json";
+  }
+
+  return fetch(baseUrl.replace(/\/+$/, "") + path, { ...init, headers: merged });
+}
+
+async function writeBytes(targetPath: string, bytes: Uint8Array): Promise<void> {
+  await mkdir(dirname(targetPath), { recursive: true });
+  await writeFile(targetPath, bytes);
+}
+
+async function downloadBytes(url: string, signal?: AbortSignal): Promise<Uint8Array> {
+  const res = await fetch(url, { signal });
+  if (!res.ok) throw new Error(`Download failed (HTTP ${res.status}): ${url}`);
+  return new Uint8Array(await res.arrayBuffer());
+}
+
+async function imageUrlToBase64(url: string): Promise<{ data: string; mimeType: string }> {
+  const bytes = await downloadBytes(url);
+  const mimeType = guessMimeFromExt(url) ?? "image/png";
+  return { data: encodeBase64(bytes), mimeType };
+}
+
+/** Lenient scan of a video task response for a downloadable URL. */
+function findVideoUrl(value: unknown): string | undefined {
+  const seen = new Set<object>();
+  const stack: unknown[] = [value];
+  while (stack.length) {
+    const cur = stack.pop();
+    if (!cur || typeof cur !== "object" || seen.has(cur as object)) continue;
+    seen.add(cur as object);
+
+    if (Array.isArray(cur)) {
+      stack.push(...cur);
+      continue;
+    }
+
+    const record = cur as Record<string, unknown>;
+    for (const key of ["url", "download_url", "video_url", "file_url", "output_url", "play_url", "link", "href"]) {
+      const v = record[key];
+      if (typeof v === "string" && /^https?:\/\//i.test(v)) return v;
+    }
+    for (const key of ["output", "outputs", "result", "results", "video", "videos", "data"]) {
+      const v = record[key];
+      if (typeof v === "string" && /^https?:\/\//i.test(v)) return v;
+      if (v && typeof v === "object") stack.push(v);
+    }
+  }
+  return undefined;
+}
+
+function extractTaskId(payload: unknown): string | undefined {
+  const p = payload as Record<string, unknown> & {
+    data?: Array<Record<string, unknown> | string>;
+  };
+  const first = p?.data?.[0];
+  if (typeof first === "string") return first;
+  if (first && typeof first === "object" && typeof first["id"] === "string") return first["id"];
+  if (typeof p?.id === "string") return p.id;
+  if (typeof p?.task_id === "string") return p.task_id;
+  if (typeof p?.video_id === "string") return p.video_id;
+  return undefined;
+}
+
+function extractVideoStatus(payload: unknown): string {
+  const p = payload as {
+    status?: unknown;
+    data?: Array<{ status?: unknown }>;
+  };
+  const raw = p?.status ?? p?.data?.[0]?.status ?? "";
+  return String(raw ?? "").toLowerCase();
+}
+
+async function resolveAudioFile(
+  input: string,
+  cwd: string,
+  signal?: AbortSignal,
+): Promise<{ bytes: Uint8Array; filename: string; mimeType: string }> {
+  if (/^https?:\/\//i.test(input)) {
+    const res = await fetch(input, { signal });
+    if (!res.ok) throw new Error(`Failed to fetch audio (HTTP ${res.status}): ${input}`);
+    const filename = basename(new URL(input).pathname) || "audio.mp3";
+    return {
+      bytes: new Uint8Array(await res.arrayBuffer()),
+      filename,
+      mimeType: guessMimeFromExt(filename) ?? "audio/mpeg",
+    };
+  }
+
+  const path = resolve(cwd, input);
+  if (!existsSync(path)) throw new Error(`Audio file not found: ${path}`);
+  const buffer = await readFile(path);
+  return {
+    bytes: new Uint8Array(buffer),
+    filename: basename(path),
+    mimeType: guessMimeFromExt(path) ?? "audio/mpeg",
+  };
+}
+
 // ── Extension Entry Point ──────────────────────────────────────────────────
 
 export default async function (pi: ExtensionAPI) {
@@ -185,15 +430,32 @@ export default async function (pi: ExtensionAPI) {
   const contextWindow = config.contextWindow ?? DEFAULTS.contextWindow;
   const maxTokens = config.maxTokens ?? DEFAULTS.maxTokens;
   const staticModels = config.models ?? [];
+  const extraHeaders = config.headers ?? {};
 
-  let discovered: ProviderModelConfig[] = [];
-  if (config.fetchModels !== false) {
+  const endpoints: EndpointsConfig = config.endpoints ?? {};
+  const embeddingsCfg = normalizeEndpoint(endpoints.embeddings);
+  const imagesCfg = normalizeEndpoint(endpoints.images);
+  const videosCfg = normalizeEndpoint(endpoints.videos);
+  const speechCfg = normalizeEndpoint(endpoints.speech);
+  const transcriptionsCfg = normalizeEndpoint(endpoints.transcriptions);
+  const responsesCfg = normalizeEndpoint(endpoints.responses);
+  const anthropicCfg = normalizeEndpoint(endpoints.anthropic);
+
+  const endpointEnabled =
+    embeddingsCfg.enabled !== false ||
+    imagesCfg.enabled !== false ||
+    videosCfg.enabled !== false ||
+    speechCfg.enabled !== false ||
+    transcriptionsCfg.enabled !== false ||
+    responsesCfg.enabled !== false ||
+    anthropicCfg.enabled !== false;
+
+  // ── Model discovery (used for both chat models and endpoint defaults) ────
+  let remote: RemoteModel[] = [];
+  const shouldFetchModels = config.fetchModels !== false || endpointEnabled;
+  if (shouldFetchModels) {
     try {
-      const remote = await fetchModelsFromEndpoint(baseUrl, apiKey);
-      discovered = remote
-        .filter((m) => m.available !== false)
-        .filter((m) => typeof m.context_window === "number" && m.context_window > 0)
-        .map((m) => mapRemoteModel(m, contextWindow, maxTokens));
+      remote = await fetchModelsFromEndpoint(baseUrl, apiKey);
     } catch (err) {
       console.warn(
         `[freellmapi] Could not fetch models from ${baseUrl}:`,
@@ -202,6 +464,18 @@ export default async function (pi: ExtensionAPI) {
     }
   }
 
+  const availableIds = remote
+    .filter((m) => m.available !== false)
+    .map((m) => m.id);
+
+  let discovered: ProviderModelConfig[] = [];
+  if (config.fetchModels !== false) {
+    discovered = remote
+      .filter((m) => m.available !== false)
+      .filter((m) => typeof m.context_window === "number" && m.context_window > 0)
+      .map((m) => mapRemoteModel(m, contextWindow, maxTokens));
+  }
+
   let models = mergeModels(staticModels, discovered);
 
   if (models.length === 0) {
@@ -217,6 +491,7 @@ export default async function (pi: ExtensionAPI) {
     }));
   }
 
+  // ── Register chat provider ───────────────────────────────────────────────
   const providerConfig: Record<string, unknown> = {
     name: config.label ?? DEFAULTS.label,
     baseUrl,
@@ -233,4 +508,393 @@ export default async function (pi: ExtensionAPI) {
   console.log(
     `[freellmapi] Registered "${providerId}" → ${baseUrl} (${models.length} model(s))`,
   );
-}
+
+  // ── Register additional API-type providers ───────────────────────────────
+  // The gateway is OpenAI-compatible but also supports the Responses and
+  // Anthropic Messages API shapes at /v1/responses and /v1/messages.
+
+  if (responsesCfg.enabled !== false) {
+    const responsesProviderId = `${providerId}-responses`;
+    pi.registerProvider(responsesProviderId, {
+      name: `${config.label ?? DEFAULTS.label} (Responses)`,
+      baseUrl,
+      apiKey,
+      api: "openai-responses",
+      models,
+      compat: { supportsDeveloperRole: false, ...(config.compat ?? {}) },
+    } as never);
+    console.log(`[freellmapi] Registered "${responsesProviderId}" → openai-responses (${models.length} model(s))`);
+  }
+
+  if (anthropicCfg.enabled !== false) {
+    const anthropicProviderId = `${providerId}-anthropic`;
+    pi.registerProvider(anthropicProviderId, {
+      name: `${config.label ?? DEFAULTS.label} (Anthropic)`,
+      baseUrl,
+      apiKey,
+      api: "anthropic-messages",
+      models,
+      compat: { supportsDeveloperRole: false, ...(config.compat ?? {}) },
+    } as never);
+    console.log(`[freellmapi] Registered "${anthropicProviderId}" → anthropic-messages (${models.length} model(s))`);
+  }
+
+  // ── Register endpoint tools ──────────────────────────────────────────────
+
+  // Default model per endpoint: explicit config wins, then discovery, then a
+  // conventional OpenAI id (which may need overriding for this gateway).
+  const embeddingsDefault =
+    embeddingsCfg.model ??
+    pickDefaultModel(availableIds, [/embed|embedding/i], "auto");
+  const imagesDefault =
+    imagesCfg.model ??
+    pickDefaultModel(availableIds, [/gemini.*image|flux|dall|sdxl|stable/i], "auto");
+  const videosDefault =
+    videosCfg.model ??
+    pickDefaultModel(availableIds, [/video|inkling|sora|veo|kling|hailuo|wan|pixverse|runway|pika|ray/i], "inkling");
+  const speechDefault =
+    speechCfg.model ??
+    pickDefaultModel(availableIds, [/tts|speech|voice|cosy|melotts|fish|bark/i], "auto");
+  const transcriptionsDefault =
+    transcriptionsCfg.model ??
+    pickDefaultModel(availableIds, [/whisper|transcri|asr|sensevoice|paraformer|audio/i], "whisper-1");
+
+  if (embeddingsCfg.enabled !== false) {
+    pi.registerTool({
+      name: "freellm_embed",
+      label: "Embed text",
+      description:
+        "Generate vector embeddings via the FreeLLM gateway (POST /embeddings). " +
+        "Useful for semantic search, clustering, similarity, or retrieval tasks.",
+      promptSnippet: "Generate text embeddings via the FreeLLM gateway (POST /embeddings)",
+      promptGuidelines: [
+        "Use freellm_embed when the task calls for semantic search, clustering, similarity, or retrieval embeddings rather than a chat completion.",
+      ],
+      parameters: Type.Object({
+        input: Type.Union([
+          Type.String({ description: "Single text to embed" }),
+          Type.Array(Type.String({ description: "Text to embed" })),
+        ]),
+        model: Type.Optional(Type.String({ description: `Embedding model id (default: ${embeddingsDefault})` })),
+        dimensions: Type.Optional(Type.Integer({ description: "Optional output dimension" })),
+        encodingFormat: Type.Optional(Type.String({ description: '"float" or "base64"' })),
+        outputFile: Type.Optional(Type.String({ description: "Path for the JSON result (default: freellm-embeddings-<ts>.json in cwd)" })),
+      }),
+      async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+        const model = params.model ?? embeddingsDefault;
+        const body: Record<string, unknown> = { model, input: params.input };
+        if (params.dimensions) body["dimensions"] = params.dimensions;
+        if (params.encodingFormat) body["encoding_format"] = params.encodingFormat;
+
+        const res = await httpRequest(baseUrl, apiKey, extraHeaders, "/embeddings", {
+          method: "POST",
+          body: JSON.stringify(body),
+          signal,
+        });
+        if (!res.ok) {
+          throw new Error(`[freellmapi] /embeddings failed (HTTP ${res.status}): ${await readErrorBody(res)}`);
+        }
+        const payload = (await res.json()) as { data?: Array<{ embedding?: unknown[] }>; model?: string; usage?: unknown };
+        const data = payload.data ?? [];
+        const dimension = data[0]?.embedding?.length ?? 0;
+        const outFile = resolve(ctx.cwd, params.outputFile ?? `freellm-embeddings-${Date.now()}.json`);
+        await writeBytes(outFile, new TextEncoder().encode(JSON.stringify(payload, null, 2)));
+
+        return {
+          content: [{
+            type: "text",
+            text: `Generated ${data.length} embedding(s) of dimension ${dimension} using model "${payload.model ?? model}". Saved to ${outFile}`,
+          }],
+          details: { outputFile: outFile, count: data.length, dimension, usage: payload.usage },
+        };
+      },
+    });
+  }
+
+  if (imagesCfg.enabled !== false) {
+    pi.registerTool({
+      name: "freellm_image",
+      label: "Generate image",
+      description:
+        "Generate images via the FreeLLM gateway (POST /images/generations). " +
+        "Returns the image(s) inline so they can be viewed and described.",
+      promptSnippet: "Generate images via the FreeLLM gateway (POST /images/generations)",
+      promptGuidelines: [
+        "Use freellm_image when the user asks to create, draw, or generate an image.",
+      ],
+      parameters: Type.Object({
+        prompt: Type.String({ description: "Image prompt" }),
+        model: Type.Optional(Type.String({ description: `Image model id (default: ${imagesDefault})` })),
+        n: Type.Optional(Type.Integer({ description: "Number of images (1-10)" })),
+        size: Type.Optional(Type.String({ description: 'e.g. "1024x1024"' })),
+        quality: Type.Optional(Type.String({ description: 'e.g. "standard" or "hd"' })),
+        style: Type.Optional(Type.String({ description: 'e.g. "vivid" or "natural"' })),
+        responseFormat: Type.Optional(Type.String({ description: '"b64_json" (default) or "url"' })),
+      }),
+      async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
+        const model = params.model ?? imagesDefault;
+        const body: Record<string, unknown> = {
+          model,
+          prompt: params.prompt,
+          n: params.n ?? 1,
+          response_format: params.responseFormat ?? "b64_json",
+        };
+        if (params.size) body["size"] = params.size;
+        if (params.quality) body["quality"] = params.quality;
+        if (params.style) body["style"] = params.style;
+
+        const res = await httpRequest(baseUrl, apiKey, extraHeaders, "/images/generations", {
+          method: "POST",
+          body: JSON.stringify(body),
+          signal,
+        });
+        if (!res.ok) {
+          throw new Error(`[freellmapi] /images/generations failed (HTTP ${res.status}): ${await readErrorBody(res)}`);
+        }
+        const payload = (await res.json()) as {
+          data?: Array<{ b64_json?: string; url?: string; revised_prompt?: string }>;
+        };
+
+        const content: ToolContent = [];
+        const images: Array<{ mimeType: string; data: string; revisedPrompt?: string }> = [];
+        const items = payload.data ?? [];
+
+        for (const item of items) {
+          let data = item.b64_json;
+          let mimeType = "image/png";
+          if (!data && item.url) {
+            try {
+              const downloaded = await imageUrlToBase64(item.url);
+              data = downloaded.data;
+              mimeType = downloaded.mimeType;
+            } catch {
+              content.push({ type: "text", text: `Image URL: ${item.url}` });
+              continue;
+            }
+          }
+          if (data) {
+            content.push({ type: "image", data, mimeType });
+            images.push({ mimeType, data, revisedPrompt: item.revised_prompt });
+          } else if (item.url) {
+            content.push({ type: "text", text: `Image URL: ${item.url}` });
+          }
+        }
+
+        const revised = images.map((i) => i.revisedPrompt).filter(Boolean);
+        const summary = `Generated ${images.length} image(s)` +
+          (revised.length ? ` (with revised prompts)` : "") +
+          (images.length !== items.length ? `; ${items.length - images.length} returned only URLs` : "");
+        content.unshift({ type: "text", text: summary });
+
+        return { content, details: { images } };
+      },
+    });
+  }
+
+  if (videosCfg.enabled !== false) {
+    pi.registerTool({
+      name: "freellm_video",
+      label: "Generate video",
+      description:
+        "Generate a video via the FreeLLM gateway (POST /videos/generations) with task polling. " +
+        "Downloads the finished video to a local file and reports the path.",
+      promptSnippet: "Generate videos via the FreeLLM gateway (POST /videos/generations)",
+      promptGuidelines: [
+        "Use freellm_video when the user asks to create or generate a video. It blocks until the render finishes, so prefer it only for explicit video requests.",
+      ],
+      parameters: Type.Object({
+        prompt: Type.String({ description: "Video prompt" }),
+        model: Type.Optional(Type.String({ description: `Video model id (default: ${videosDefault})` })),
+        n: Type.Optional(Type.Integer({ description: "Number of videos" })),
+        size: Type.Optional(Type.String({ description: 'e.g. "1280x720"' })),
+        seconds: Type.Optional(Type.String({ description: 'e.g. "5"' })),
+        pollTimeoutSeconds: Type.Optional(Type.Number({ description: "Max time to wait for completion (default 300)" })),
+      }),
+      async execute(_toolCallId, params, signal, onUpdate, ctx) {
+        const model = params.model ?? videosDefault;
+        const body: Record<string, unknown> = { model, prompt: params.prompt };
+        if (params.n) body["n"] = params.n;
+        if (params.size) body["size"] = params.size;
+        if (params.seconds) body["seconds"] = params.seconds;
+
+        const submit = await httpRequest(baseUrl, apiKey, extraHeaders, "/videos/generations", {
+          method: "POST",
+          body: JSON.stringify(body),
+          signal,
+        });
+        if (!submit.ok) {
+          throw new Error(`[freellmapi] /videos/generations failed (HTTP ${submit.status}): ${await readErrorBody(submit)}`);
+        }
+        const submitted = (await submit.json()) as unknown;
+
+        const taskId = extractTaskId(submitted);
+        if (!taskId) {
+          const directUrl = findVideoUrl(submitted);
+          if (directUrl) {
+            const bytes = await downloadBytes(directUrl, signal);
+            const outFile = resolve(ctx.cwd, `freellm-video-${Date.now()}${guessVideoExt(directUrl)}`);
+            await writeBytes(outFile, bytes);
+            return {
+              content: [{ type: "text", text: `Video ready and downloaded (${bytes.length} bytes) to ${outFile}. Source URL: ${directUrl}` }],
+              details: { outputFile: outFile, url: directUrl },
+            };
+          }
+          throw new Error(`[freellmapi] /videos/generations returned no task id: ${JSON.stringify(submitted).slice(0, 500)}`);
+        }
+
+        const timeoutSeconds = params.pollTimeoutSeconds ?? 300;
+        const deadline = Date.now() + timeoutSeconds * 1000;
+        let task: unknown = submitted;
+        let status = extractVideoStatus(task);
+
+        while (!VIDEO_TERMINAL_STATUSES.has(status) && Date.now() < deadline) {
+          if (signal?.aborted) throw new Error("Video generation cancelled");
+          await sleep(3000);
+          const poll = await httpRequest(baseUrl, apiKey, extraHeaders, `/videos/${encodeURIComponent(taskId)}`, { signal });
+          if (!poll.ok) {
+            status = `error (poll HTTP ${poll.status})`;
+            break;
+          }
+          task = (await poll.json()) as unknown;
+          status = extractVideoStatus(task);
+          if (!VIDEO_TERMINAL_STATUSES.has(status)) {
+            onUpdate?.({
+              content: [{ type: "text", text: `Video ${taskId} status: ${status || "processing"}…` }],
+            });
+          }
+        }
+
+        if (!VIDEO_SUCCESS_STATUSES.has(status)) {
+          throw new Error(
+            `[freellmapi] Video generation did not complete (status: ${status || "unknown"}). Last response: ${JSON.stringify(task).slice(0, 500)}`,
+          );
+        }
+
+        const url = findVideoUrl(task);
+        if (!url) {
+          throw new Error(`[freellmapi] Video completed but no URL found: ${JSON.stringify(task).slice(0, 1000)}`);
+        }
+
+        const bytes = await downloadBytes(url, signal);
+        const outFile = resolve(ctx.cwd, `freellm-video-${taskId}${guessVideoExt(url)}`);
+        await writeBytes(outFile, bytes);
+
+        return {
+          content: [{
+            type: "text",
+            text: `Video ready (status ${status}). Downloaded ${bytes.length} bytes to ${outFile}. Source URL: ${url}`,
+          }],
+          details: { taskId, status, outputFile: outFile, url },
+        };
+      },
+    });
+  }
+
+  if (speechCfg.enabled !== false) {
+    pi.registerTool({
+      name: "freellm_speech",
+      label: "Text to speech",
+      description:
+        "Synthesize speech from text via the FreeLLM gateway (POST /audio/speech). " +
+        "Writes the audio to a local file and reports the path.",
+      promptSnippet: "Synthesize speech via the FreeLLM gateway (POST /audio/speech)",
+      promptGuidelines: [
+        "Use freellm_speech when the user asks to generate, read aloud, or synthesize speech/audio from text.",
+      ],
+      parameters: Type.Object({
+        text: Type.String({ description: "Text to synthesize" }),
+        model: Type.Optional(Type.String({ description: `TTS model id (default: ${speechDefault})` })),
+        voice: Type.Optional(Type.String({ description: 'Voice name (default: "alloy")' })),
+        speed: Type.Optional(Type.Number({ description: "Playback speed (0.25-4.0)" })),
+        format: Type.Optional(Type.String({ description: 'Audio format: "mp3" (default), "opus", "aac", "flac", "wav", "pcm"' })),
+        outputFile: Type.Optional(Type.String({ description: `Output path (default: freellm-speech-<ts>.<format> in cwd)` })),
+      }),
+      async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+        const model = params.model ?? speechDefault;
+        const format = params.format ?? "mp3";
+        const body: Record<string, unknown> = {
+          model,
+          input: params.text,
+          voice: params.voice ?? "alloy",
+          response_format: format,
+        };
+        if (params.speed != null) body["speed"] = params.speed;
+
+        const res = await httpRequest(baseUrl, apiKey, extraHeaders, "/audio/speech", {
+          method: "POST",
+          body: JSON.stringify(body),
+          signal,
+        });
+        if (!res.ok) {
+          throw new Error(`[freellmapi] /audio/speech failed (HTTP ${res.status}): ${await readErrorBody(res)}`);
+        }
+
+        const bytes = new Uint8Array(await res.arrayBuffer());
+        const outFile = resolve(ctx.cwd, params.outputFile ?? `freellm-speech-${Date.now()}.${format}`);
+        await writeBytes(outFile, bytes);
+
+        return {
+          content: [{
+            type: "text",
+            text: `Synthesized ${bytes.length} bytes of ${format} audio with voice "${params.voice ?? "alloy"}" → ${outFile}`,
+          }],
+          details: { outputFile: outFile, format, bytes: bytes.length },
+        };
+      },
+    });
+  }
+
+  if (transcriptionsCfg.enabled !== false) {
+    pi.registerTool({
+      name: "freellm_transcribe",
+      label: "Transcribe audio",
+      description:
+        "Transcribe an audio file via the FreeLLM gateway (POST /audio/transcriptions, multipart upload). " +
+        "Accepts a local path or an http(s) URL.",
+      promptSnippet: "Transcribe audio via the FreeLLM gateway (POST /audio/transcriptions)",
+      promptGuidelines: [
+        "Use freellm_transcribe when the user asks to transcribe, caption, or get the text of an audio recording.",
+      ],
+      parameters: Type.Object({
+        file: Type.String({ description: "Local path or http(s) URL of the audio file" }),
+        model: Type.Optional(Type.String({ description: `Transcription model id (default: ${transcriptionsDefault})` })),
+        language: Type.Optional(Type.String({ description: 'Optional ISO-639-1 language, e.g. "en"' })),
+        format: Type.Optional(Type.String({ description: 'Response format: "json" (default), "text", "srt", "verbose_json", "vtt"' })),
+        prompt: Type.Optional(Type.String({ description: "Optional hint to guide transcription style/vocabulary" })),
+      }),
+      async execute(_toolCallId, params, signal, _onUpdate, ctx) {
+        const model = params.model ?? transcriptionsDefault;
+        const { bytes, filename, mimeType } = await resolveAudioFile(params.file, ctx.cwd, signal);
+
+        const form = new FormData();
+        form.append("file", new Blob([bytes as BlobPart], { type: mimeType }), filename);
+        form.append("model", model);
+        if (params.language) form.append("language", params.language);
+        if (params.format) form.append("response_format", params.format);
+        if (params.prompt) form.append("prompt", params.prompt);
+
+        const res = await httpRequest(baseUrl, apiKey, extraHeaders, "/audio/transcriptions", {
+          method: "POST",
+          body: form,
+          signal,
+        });
+        if (!res.ok) {
+          throw new Error(`[freellmapi] /audio/transcriptions failed (HTTP ${res.status}): ${await readErrorBody(res)}`);
+        }
+
+        const raw = await res.text();
+        let text = raw;
+        if (params.format === undefined || params.format === "json" || params.format === "verbose_json") {
+          try {
+            const json = JSON.parse(raw) as { text?: string };
+            text = json.text ?? raw;
+          } catch {
+            text = raw;
+          }
+        }
+
+        return { content: [{ type: "text", text }], details: { raw } };
+      },
+    });
+  }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pi-provider-freellmapi",
-  "version": "1.0.2",
-  "description": "Register the FreeLLM API gateway (freeapi.n.cofire.cn) as an OpenAI-compatible provider in pi, with automatic model discovery",
+  "version": "1.0.4",
+  "description": "Register the FreeLLM API gateway (freeapi.n.cofire.cn) as an OpenAI-compatible provider in pi, with automatic model discovery and tools for embeddings, image/video generation, speech, and transcription",
   "keywords": [
     "pi",
     "pi-package",