qlogicagent 2.13.4 → 2.13.5

package/dist/cli/media-runtime-facade.js

@@ -0,0 +1,389 @@
+import { IMAGE_GENERATE_TOOL_NAME, IMAGE_GENERATE_TOOL_SCHEMA } from "../skills/tools/image-generate-tool.js";
+import { AsyncLocalStorage } from "node:async_hooks";
+import { MUSIC_GENERATE_TOOL_NAME, MUSIC_GENERATE_TOOL_SCHEMA } from "../skills/tools/music-generate-tool.js";
+import { TTS_TOOL_NAME, TTS_TOOL_SCHEMA } from "../skills/tools/tts-tool.js";
+import { VIDEO_GENERATE_TOOL_NAME, VIDEO_GENERATE_TOOL_SCHEMA } from "../skills/tools/video-generate-tool.js";
+import { isMiniMaxLyricsTransport, resolveMediaApiKey, } from "./provider-core-facade.js";
+import { createLegacyToolCatalog } from "./tool-registry-adapter.js";
+import { buildMediaToolParameterSchema, normalizeMediaCapabilitySchema, resolveMediaCapabilityDecision, } from "./media-capability-schema.js";
+let mediaClient;
+let mediaApiKeys = {};
+let mediaUsageCallback;
+let mediaCapabilityCheckCallback;
+let mediaProgressCallback;
+let mediaProviders = {};
+let providerToolAPI;
+const mediaOutputDirStorage = new AsyncLocalStorage();
+export function setProviderToolAPI(api) {
+    providerToolAPI = api;
+}
+export function getProviderToolAPI() {
+    return providerToolAPI;
+}
+export function setMediaClientConfig(client, apiKeys, onMediaUsage, providers, onMediaProgress, onMediaCapabilityCheck) {
+    mediaClient = client;
+    mediaApiKeys = apiKeys ?? {};
+    mediaUsageCallback = onMediaUsage;
+    mediaProgressCallback = onMediaProgress;
+    mediaCapabilityCheckCallback = onMediaCapabilityCheck;
+    mediaProviders = providers ?? {};
+    updateAllMediaToolDescriptions();
+}
+export function getMediaRuntimeContext() {
+    return { mediaClient, mediaProviders, mediaApiKeys };
+}
+export async function runWithMediaOutputDir(outputDir, fn) {
+    return mediaOutputDirStorage.run(outputDir, fn);
+}
+function resolveDesignated(type) {
+    const designated = mediaProviders[type];
+    if (!designated || !mediaClient)
+        return undefined;
+    return mediaClient.resolveModelById(designated.provider, designated.model, type);
+}
+function updateAllMediaToolDescriptions() {
+    updateVideoToolDescription();
+    updateImageToolDescription();
+    updateTtsToolDescription();
+    updateMusicToolDescription();
+}
+function updateVideoToolDescription() {
+    const tool = createLegacyToolCatalog().findTool(VIDEO_GENERATE_TOOL_NAME);
+    if (!tool)
+        return;
+    const designated = mediaProviders.video;
+    if (!designated || !mediaClient)
+        return;
+    const resolved = mediaClient.resolveModelById(designated.provider, designated.model, "video");
+    if (!resolved)
+        return;
+    const capability = normalizeMediaCapabilitySchema("video", resolved.modelInfo.mediaCapabilities);
+    const maxDur = capability.limits.durationSeconds?.max ?? 10;
+    const minDur = maxDur >= 10 ? 4 : 3;
+    const resolutions = capability.limits.resolutions?.join("/") || "720P";
+    const modelName = resolved.modelInfo.name || resolved.modelInfo.id;
+    tool.description =
+        "Generate a short video clip from a text prompt, reference image, source video, or multimodal references. " +
+            `Current model: ${modelName}. Single-shot: ${minDur}-${maxDur}s. ` +
+            `For longer videos (>${maxDur}s), use multi-shot storyboard workflow. ` +
+            "Prompt MUST be in English. ALL image/video/audio URLs MUST be publicly accessible HTTP/HTTPS URLs.";
+    tool.parameters = buildMediaToolParameterSchema(VIDEO_GENERATE_TOOL_NAME, VIDEO_GENERATE_TOOL_SCHEMA, capability);
+    const props = tool.parameters?.properties;
+    const durationProp = props?.duration;
+    if (durationProp) {
+        durationProp.minimum = minDur;
+        durationProp.maximum = maxDur;
+        durationProp.description =
+            `Video duration in seconds (${minDur}-${maxDur}s for ${modelName}). ` +
+                `For longer videos (>${maxDur}s), use multi-shot storyboard workflow (generate + extend/merge).`;
+    }
+    const resProp = props?.resolution;
+    if (resProp && capability.limits.resolutions?.length) {
+        resProp.description = `Output resolution: ${resolutions}. Default: '720p'.`;
+    }
+}
+function updateImageToolDescription() {
+    const tool = createLegacyToolCatalog().findTool(IMAGE_GENERATE_TOOL_NAME);
+    if (!tool)
+        return;
+    const resolved = resolveDesignated("image");
+    if (!resolved)
+        return;
+    const modelName = resolved.modelInfo.name || resolved.modelInfo.id;
+    const capability = normalizeMediaCapabilitySchema("image", resolved.modelInfo.mediaCapabilities);
+    const sizeParts = [];
+    if (capability.limits.sizes?.length)
+        sizeParts.push(`Sizes: ${capability.limits.sizes.join(", ")}`);
+    if (capability.features.transparentBackground)
+        sizeParts.push("Supports transparent background");
+    tool.description =
+        "Generate images from a text prompt. Enrich vague prompts with style, lighting, composition, color, and mood before calling. " +
+            "Supports img2img via image_url reference. " +
+            `Current model: ${modelName}. ${sizeParts.join(". ")}${sizeParts.length ? ". " : ""}` +
+            "ALL reference image URLs MUST be publicly accessible HTTP/HTTPS URLs; use file_upload first for local files.";
+    tool.parameters = buildMediaToolParameterSchema(IMAGE_GENERATE_TOOL_NAME, IMAGE_GENERATE_TOOL_SCHEMA, capability);
+    if (capability.limits.sizes?.length) {
+        const props = tool.parameters?.properties;
+        const sizeProp = props?.size;
+        if (sizeProp) {
+            sizeProp.description = `Dimensions: ${capability.limits.sizes.join(", ")}. Default: '1024x1024'. Auto-infer from purpose.`;
+        }
+    }
+}
+function updateTtsToolDescription() {
+    const tool = createLegacyToolCatalog().findTool(TTS_TOOL_NAME);
+    if (!tool)
+        return;
+    const resolved = resolveDesignated("tts");
+    if (!resolved)
+        return;
+    const modelName = resolved.modelInfo.name || resolved.modelInfo.id;
+    const capability = normalizeMediaCapabilitySchema("tts", resolved.modelInfo.mediaCapabilities);
+    const infoParts = [];
+    if (capability.limits.voices?.length)
+        infoParts.push(`Voices: ${capability.limits.voices.join(", ")}`);
+    if (capability.limits.maxCharacters)
+        infoParts.push(`Max: ${capability.limits.maxCharacters} characters per request`);
+    if (capability.limits.formats?.length)
+        infoParts.push(`Formats: ${capability.limits.formats.join(", ")}`);
+    tool.description =
+        "Convert text to speech (TTS) as spoken audio. Use for narration or voice messages, not music. " +
+            `Current model: ${modelName}. ${infoParts.join(". ")}${infoParts.length ? "." : ""}`;
+    tool.parameters = buildMediaToolParameterSchema(TTS_TOOL_NAME, TTS_TOOL_SCHEMA, capability);
+    if (capability.limits.voices?.length) {
+        const props = tool.parameters?.properties;
+        const voiceProp = props?.voice;
+        if (voiceProp)
+            voiceProp.description = `Voice name. Available: ${capability.limits.voices.join(", ")}.`;
+    }
+}
+function updateMusicToolDescription() {
+    const tool = createLegacyToolCatalog().findTool(MUSIC_GENERATE_TOOL_NAME);
+    if (!tool)
+        return;
+    const resolved = resolveDesignated("music");
+    if (!resolved)
+        return;
+    const modelName = resolved.modelInfo.name || resolved.modelInfo.id;
+    const capability = normalizeMediaCapabilitySchema("music", resolved.modelInfo.mediaCapabilities);
+    const infoParts = [];
+    if (capability.limits.durationSeconds?.max)
+        infoParts.push(`Max duration: ${capability.limits.durationSeconds.max}s`);
+    if (capability.limits.formats?.length)
+        infoParts.push(`Formats: ${capability.limits.formats.join(", ")}`);
+    tool.description =
+        "Generate music, songs, or melodies from text descriptions. Prompt MUST be in English. " +
+            "Supports lyrics for vocal songs; use tts for spoken-word audio. " +
+            `Current model: ${modelName}. ${infoParts.join(". ")}${infoParts.length ? ". " : ""}`;
+    tool.parameters = buildMediaToolParameterSchema(MUSIC_GENERATE_TOOL_NAME, MUSIC_GENERATE_TOOL_SCHEMA, capability);
+    if (capability.limits.durationSeconds?.max) {
+        const props = tool.parameters?.properties;
+        const durationProp = props?.duration;
+        if (durationProp) {
+            durationProp.maximum = capability.limits.durationSeconds.max;
+            durationProp.description =
+                `Duration in seconds (5-${capability.limits.durationSeconds.max}s for ${modelName}). ` +
+                    "Infer from purpose: ringtone 15-30s, BGM 30-60s, full song 120-180s.";
+        }
+    }
+}
+export async function generateMedia(request) {
+    if (!mediaClient) {
+        throw new Error(`No media provider configured for ${request.mediaType}`);
+    }
+    const designated = mediaProviders[request.mediaType];
+    const providerId = designated?.provider;
+    const modelId = designated?.model || request.model;
+    if (!providerId) {
+        throw new Error(`No provider configured for ${request.mediaType}. Please configure a provider in settings.`);
+    }
+    const requestWithProgress = { ...request };
+    if (request.mediaType === "image") {
+        // Some image models (Volcengine Seedream 4.5/5.0) reject sub-~3.7MP sizes
+        // with a hard 400. The image tool suggests 1024-class sizes → every gen
+        // fails (the turn appears to hang on repeated failures). Bump sub-minimum
+        // sizes up to a valid resolution, preserving aspect ratio.
+        requestWithProgress.size = normalizeImageSize(request.size, providerId, modelId || request.model);
+    }
+    if (mediaProgressCallback && !requestWithProgress.onProgress) {
+        requestWithProgress.onProgress = (percent, status, taskId) => {
+            mediaProgressCallback(taskId ?? "pending", request.mediaType, percent, status, providerId);
+        };
+    }
+    const resolvedModel = mediaClient.resolveModelById(providerId, modelId || request.model, request.mediaType);
+    const capability = normalizeMediaCapabilitySchema(request.mediaType, resolvedModel?.modelInfo.mediaCapabilities);
+    const capabilityDecision = resolveMediaCapabilityDecision(requestWithProgress, capability);
+    emitMediaCapabilityCheck(providerId, modelId || request.model, request.mediaType, capability, capabilityDecision);
+    if (!capabilityDecision.allowed) {
+        throw new Error(`Configured media model ${providerId}/${modelId || request.model} cannot handle ${request.mediaType} request: ` +
+            capabilityDecision.reason);
+    }
+    const primaryResult = await tryGenerateWithProvider(providerId, modelId || request.model, requestWithProgress);
+    if (primaryResult.ok)
+        return primaryResult.result;
+    throw new Error(`Configured media provider ${providerId} failed for ${request.mediaType}: ${primaryResult.error}`);
+}
+function emitMediaCapabilityCheck(provider, model, mediaType, capability, decision) {
+    mediaCapabilityCheckCallback?.({
+        mediaType,
+        provider,
+        model,
+        mode: decision.mode,
+        allowed: decision.allowed,
+        ...(!decision.allowed ? { reason: decision.reason } : {}),
+        capabilities: {
+            modes: capability.modes,
+            source: capability.source,
+            limits: capability.limits,
+            features: capability.features,
+        },
+    });
+}
+async function tryGenerateWithProvider(providerId, modelId, request) {
+    const transport = mediaClient.getTransport(providerId);
+    if (!transport)
+        return { ok: false, error: "transport not available" };
+    const apiKey = resolveMediaApiKey(mediaApiKeys, providerId);
+    if (!apiKey)
+        return { ok: false, error: "no API key" };
+    try {
+        const outputDir = mediaOutputDirStorage.getStore();
+        const finalRequest = {
+            ...request,
+            model: modelId,
+            ...(outputDir ? { outputDir } : {}),
+        };
+        const result = await transport.generate(finalRequest, apiKey);
+        if (requiresMediaUrl(request.mediaType) && result.mediaUrls.length === 0) {
+            return { ok: false, error: `${providerId}/${modelId} returned no media URLs` };
+        }
+        if (!result.billingUnit) {
+            if (request.mediaType === "tts" && request.text) {
+                result.billingUnit = "per_character";
+                result.billingQuantity = request.text.length;
+            }
+            else if ((request.mediaType === "video" || request.mediaType === "music") && request.duration) {
+                result.billingUnit = "per_second";
+                result.billingQuantity = request.duration;
+            }
+            else {
+                result.billingUnit = "per_call";
+                result.billingQuantity = 1;
+            }
+        }
+        // Image generation is priced per generated image upstream (llmrouter
+        // PER_IMAGE / quantity_key "images"). Force this even when a transport
+        // already labelled the result "per_call" (the Volcengine Seedream transport
+        // does) — otherwise the cloud ledger reads {calls} and prices it at ¥0.
+        // Prefer the transport's reported image count (metadata.billableUnits.images).
+        // The published provider-core BillingUnit union predates "per_image", so the
+        // literal is assigned through a cast (forward-compat with the cloud pricing).
+        if (request.mediaType === "image") {
+            const reportedImages = result
+                .metadata?.billableUnits?.images;
+            result.billingUnit = "per_image";
+            result.billingQuantity = reportedImages ?? result.billingQuantity ?? Math.max(1, result.mediaUrls?.length ?? 1);
+        }
+        const model = result.model ?? finalRequest.model;
+        const unit = result.billingUnit ?? "per_call";
+        const qty = result.billingQuantity ?? 1;
+        result.provider = providerId;
+        result.model = model;
+        result.billingUnit = unit;
+        result.billingQuantity = qty;
+        mediaUsageCallback?.({
+            mediaType: request.mediaType,
+            provider: providerId,
+            model,
+            billingUnit: unit,
+            billingQuantity: qty,
+        });
+        return { ok: true, result };
+    }
+    catch (err) {
+        return { ok: false, error: err instanceof Error ? err.message : String(err) };
+    }
+}
+function requiresMediaUrl(mediaType) {
+    return mediaType === "image"
+        || mediaType === "video"
+        || mediaType === "music"
+        || mediaType === "music_realtime"
+        || mediaType === "tts"
+        || mediaType === "3d";
+}
+// Volcengine Seedream 4.5/5.0 reject images below this many pixels with a 400.
+const SEEDREAM_MIN_PIXELS = 3_686_400; // = 1920×1920 (2K class)
+/**
+ * Ensure an image size satisfies the resolved model's minimum resolution.
+ * Only applies to models with a known minimum (Volcengine Seedream); everything
+ * else is returned unchanged so providers that require small sizes (e.g. OpenAI
+ * 1024×1024) keep working. Sub-minimum "WxH" sizes are scaled up preserving
+ * aspect ratio; anything unparseable falls back to the always-valid named "2K".
+ */
+export function normalizeImageSize(size, providerId, modelId) {
+    const needsHiRes = providerId === "volcengine" && /seedream/i.test(modelId ?? "");
+    if (!needsHiRes)
+        return size;
+    if (!size)
+        return size; // undefined → transport defaults to "2K" (already valid)
+    const match = /^\s*(\d+)\s*[x×]\s*(\d+)\s*$/i.exec(size);
+    if (!match) {
+        // Named/unknown size ("1K", "1024", …). Keep 2K/4K; bump anything else.
+        return /^\s*(2k|4k)\s*$/i.test(size) ? size : "2K";
+    }
+    const w = Number(match[1]);
+    const h = Number(match[2]);
+    if (!(w > 0 && h > 0))
+        return "2K";
+    if (w * h >= SEEDREAM_MIN_PIXELS)
+        return size; // already valid
+    const scale = Math.sqrt((SEEDREAM_MIN_PIXELS * 1.06) / (w * h));
+    const snap = (n) => Math.max(64, Math.round((n * scale) / 64) * 64);
+    const nw = snap(w);
+    const nh = snap(h);
+    return nw * nh >= SEEDREAM_MIN_PIXELS ? `${nw}x${nh}` : "2K";
+}
+export function isMediaAvailable(mediaType) {
+    if (!mediaClient)
+        return false;
+    const designated = mediaProviders[mediaType];
+    if (!designated)
+        return false;
+    const transport = mediaClient.getTransport(designated.provider);
+    const apiKey = resolveMediaApiKey(mediaApiKeys, designated.provider);
+    return !!(transport && apiKey);
+}
+export function isMediaOperationAvailable(mediaType, operation) {
+    if (!isMediaAvailable(mediaType))
+        return false;
+    const designated = mediaProviders[mediaType];
+    const resolved = mediaClient.resolveModelById(designated.provider, designated.model, mediaType);
+    if (!resolved)
+        return false;
+    const capability = normalizeMediaCapabilitySchema(mediaType, resolved.modelInfo.mediaCapabilities);
+    return capability.modes.includes(operation);
+}
+export function isMediaOperationDeclaredAvailable(mediaType, operation) {
+    if (!isMediaAvailable(mediaType))
+        return false;
+    const designated = mediaProviders[mediaType];
+    const resolved = mediaClient.resolveModelById(designated.provider, designated.model, mediaType);
+    if (!resolved)
+        return false;
+    const capability = normalizeMediaCapabilitySchema(mediaType, resolved.modelInfo.mediaCapabilities);
+    return capability.declaredModes.includes(operation);
+}
+export async function generateMusicLyrics(prompt) {
+    const musicProvider = mediaProviders.music;
+    if (!musicProvider || !mediaClient)
+        return "";
+    const transport = mediaClient.getTransport(musicProvider.provider);
+    if (!isMiniMaxLyricsTransport(transport))
+        return "";
+    const apiKey = resolveMediaApiKey(mediaApiKeys, musicProvider.provider);
+    if (!apiKey)
+        return "";
+    return transport.generateLyrics(prompt, apiKey);
+}
+export async function cancelMediaTask(params) {
+    const providerId = params.provider ?? "doubao";
+    if (!mediaClient)
+        return { ok: false, message: "Media client not configured." };
+    const transport = mediaClient.getTransport(providerId);
+    if (!transport)
+        return { ok: false, message: `No transport for provider: ${providerId}` };
+    const apiKey = resolveMediaApiKey(mediaApiKeys, providerId);
+    if (!apiKey)
+        return { ok: false, message: `No API key for provider: ${providerId}` };
+    try {
+        if ("deleteVideoTask" in transport && typeof transport.deleteVideoTask === "function") {
+            await transport.deleteVideoTask(params.taskId, apiKey);
+            return { ok: true, message: "Task cancelled." };
+        }
+        return { ok: false, message: `Provider ${providerId} does not support task cancellation.` };
+    }
+    catch (err) {
+        return { ok: false, message: err instanceof Error ? err.message : String(err) };
+    }
+}

package/dist/cli/media-runtime-facade.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/cli/media-runtime-facade.test.js

@@ -0,0 +1,42 @@
+/**
+ * Image size normalization for models with a minimum resolution requirement.
+ * Volcengine Seedream 4.5/5.0 reject sub-~3.7MP images with a hard 400 — the
+ * image tool's 1024-class suggestions made every 对话生图 fail. normalizeImageSize
+ * bumps sub-minimum sizes up (aspect-ratio preserved) only for those models.
+ */
+import { describe, it, expect } from "vitest";
+import { normalizeImageSize } from "./media-runtime-facade.js";
+const MIN = 3_686_400;
+function pixels(size) {
+    if (!size)
+        return 0;
+    const m = /^(\d+)x(\d+)$/.exec(size);
+    return m ? Number(m[1]) * Number(m[2]) : 0;
+}
+describe("normalizeImageSize (Seedream minimum resolution)", () => {
+    it("bumps sub-minimum square sizes up to a valid resolution for Seedream", () => {
+        const out = normalizeImageSize("1024x1024", "volcengine", "doubao-seedream-4-5");
+        expect(out).not.toBe("1024x1024");
+        expect(pixels(out)).toBeGreaterThanOrEqual(MIN);
+    });
+    it("bumps sub-minimum non-square sizes while keeping them above the minimum", () => {
+        for (const s of ["1792x1024", "1024x1792"]) {
+            const out = normalizeImageSize(s, "volcengine", "doubao-seedream-4-5");
+            expect(pixels(out)).toBeGreaterThanOrEqual(MIN);
+        }
+    });
+    it("leaves already-valid sizes untouched", () => {
+        expect(normalizeImageSize("2048x2048", "volcengine", "doubao-seedream-5-0-lite")).toBe("2048x2048");
+        expect(normalizeImageSize("2K", "volcengine", "doubao-seedream-4-5")).toBe("2K");
+    });
+    it("maps small named sizes to the always-valid 2K", () => {
+        expect(normalizeImageSize("1K", "volcengine", "doubao-seedream-4-5")).toBe("2K");
+    });
+    it("does NOT touch sizes for other providers/models (e.g. OpenAI needs 1024)", () => {
+        expect(normalizeImageSize("1024x1024", "openai", "dall-e-3")).toBe("1024x1024");
+        expect(normalizeImageSize("1024x1024", "volcengine", "doubao-seedance-2-0")).toBe("1024x1024");
+    });
+    it("leaves undefined undefined (transport defaults to a valid 2K)", () => {
+        expect(normalizeImageSize(undefined, "volcengine", "doubao-seedream-4-5")).toBeUndefined();
+    });
+});

package/dist/cli/memory-candidate-service.d.ts

@@ -0,0 +1,83 @@
+interface MemoryLlmClient {
+    model: string;
+    apiKey: string;
+    transport: {
+        stream(request: Record<string, unknown>, apiKey: string): AsyncIterable<{
+            type: string;
+            text?: string;
+        }>;
+    };
+}
+/** Model purposes the memory pipeline can resolve (text + multimodal). */
+export type MemoryModelPurpose = "smallModel" | "textGeneration" | "imageUnderstanding" | "stt" | "videoUnderstanding";
+export interface MemoryCandidateHost {
+    memoryProvider?: {
+        search?(query: string, userId: string, options?: {
+            limit?: number;
+            minScore?: number;
+        }): Promise<Array<{
+            blockId: string;
+            text: string;
+            score?: number;
+            metadata?: Record<string, unknown>;
+        }>>;
+        /** Read an uploaded attachment's bytes (for multimodal understanding). */
+        readAttachmentBytes?(id: string): Promise<{
+            bytes: Buffer;
+            mimeType: string;
+        } | null>;
+        /** Persist the understood/extracted text back on the attachment row. */
+        setAttachmentText?(id: string, text: string): void;
+    } | null;
+    resolveClientForPurpose?(purpose: MemoryModelPurpose): MemoryLlmClient | null;
+}
+export interface MemoryAttachment {
+    type: string;
+    filename: string;
+    mimeType: string;
+    text: string;
+    url: string;
+    path: string;
+    size?: number;
+}
+export interface RelatedMemoryHit {
+    id: string;
+    text: string;
+    score: number;
+    category?: string;
+}
+export interface MemoryCandidate {
+    id: string;
+    text: string;
+    category: string;
+    importance: number;
+    confidence: number;
+    source: string;
+    eventDate: string;
+    tags: string[];
+    relatedMemoryIds: string[];
+    evidence: Array<Record<string, unknown>>;
+    requiresConfirmation: boolean;
+    reason: string;
+}
+export interface MemoryCandidateParams {
+    text?: string;
+    attachments?: unknown[];
+    tags?: unknown;
+    category?: unknown;
+    importance?: unknown;
+    source?: unknown;
+    autoExtract?: boolean;
+}
+export declare function normalizeInputCandidate(input: unknown): MemoryCandidate | null;
+export declare function buildMemoryCandidates(ctx: MemoryCandidateHost, params: MemoryCandidateParams, userId: string): Promise<{
+    candidates: MemoryCandidate[];
+    related: RelatedMemoryHit[];
+    source: "llm" | "deterministic";
+}>;
+export declare function buildMemoryCandidateProposal(ctx: MemoryCandidateHost, params: MemoryCandidateParams, userId: string): Promise<{
+    candidates: MemoryCandidate[];
+    relatedMemories: RelatedMemoryHit[];
+    extractor: "llm" | "deterministic";
+}>;
+export {};