vidspotai-shared 1.0.67 → 1.0.69-dev.0

package/lib/services/agent/llmCallerGateway.js

@@ -0,0 +1,360 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GatewayLlmCaller = void 0;
+const zod_1 = require("zod");
+/**
+ * Marks a transient error that should trigger HTTP retry (and, when exhausted,
+ * model fallback). Non-transient errors (4xx other than 408/429, schema
+ * validation, malformed JSON) are NOT wrapped and propagate immediately.
+ */
+class TransientLlmError extends Error {
+    constructor(message, cause) {
+        super(message);
+        this.cause = cause;
+        this.name = "TransientLlmError";
+    }
+}
+const TRANSIENT_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
+function isTransientNetworkError(err) {
+    if (!err || typeof err !== "object")
+        return false;
+    const e = err;
+    const code = e.code ?? e.cause?.code;
+    if (code && /^(ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|EPIPE|UND_ERR_.*)$/.test(code)) {
+        return true;
+    }
+    // Our own per-request timeout (ctrl.abort() in postOnce / structuredStream)
+    // surfaces as AbortError "This operation was aborted". Treat as transient so
+    // the retry loop (and then the fallback model) gets a chance — otherwise a
+    // single slow Gemini Flash vision call sinks the entire critic stage with
+    // no recovery.
+    if (e.name === "AbortError")
+        return true;
+    const msg = String(e.message ?? "");
+    return /fetch failed|socket hang up|other side closed|network|timeout|operation was aborted/i.test(msg);
+}
+class GatewayLlmCaller {
+    constructor(cfg) {
+        this.cfg = cfg;
+        this.fetchImpl = cfg.fetchImpl ?? fetch;
+        this.timeoutMs = cfg.timeoutMs ?? 60000;
+        this.maxRetries = cfg.maxRetries ?? 3;
+        this.retryBaseMs = cfg.retryBaseMs ?? 250;
+    }
+    async chat(req) {
+        const attempted = [];
+        const run = async (model) => {
+            attempted.push(model);
+            const messages = await Promise.all(req.messages.map((m) => this.encodeMessage(m)));
+            const raw = await this.postWithRetry({
+                model: this.modelId(model),
+                messages,
+                temperature: req.temperature,
+                max_tokens: req.maxTokens,
+                stop: req.stop,
+            });
+            const text = raw.choices?.[0]?.message?.content ?? "";
+            return {
+                text: typeof text === "string" ? text : JSON.stringify(text),
+                usage: this.usage(raw),
+                requestId: raw.id,
+            };
+        };
+        try {
+            const out = await run(req.model);
+            return { ...out, attemptedModels: attempted, usedFallback: false };
+        }
+        catch (err) {
+            if (req.fallbackModel && err instanceof TransientLlmError) {
+                const out = await run(req.fallbackModel);
+                return { ...out, attemptedModels: attempted, usedFallback: true };
+            }
+            throw err;
+        }
+    }
+    async structured(req) {
+        const attempted = [];
+        const jsonSchema = zod_1.z.toJSONSchema(req.schema, { target: "draft-7" });
+        const run = async (model) => {
+            attempted.push(model);
+            const messages = await Promise.all(req.messages.map((m) => this.encodeMessage(m)));
+            const raw = await this.postWithRetry({
+                model: this.modelId(model),
+                messages,
+                temperature: req.temperature,
+                max_tokens: req.maxTokens,
+                response_format: {
+                    type: "json_schema",
+                    json_schema: {
+                        name: req.schemaName,
+                        schema: jsonSchema,
+                        strict: true,
+                    },
+                },
+            });
+            const txt = raw.choices?.[0]?.message?.content ?? "";
+            if (!txt || typeof txt !== "string") {
+                // Empty body from the model is not retryable across providers —
+                // surface immediately; planner-level retry can re-prompt.
+                throw new Error(`GatewayLlmCaller.structured: empty response for schema ${req.schemaName}`);
+            }
+            let parsed;
+            try {
+                parsed = JSON.parse(txt);
+            }
+            catch {
+                throw new Error(`GatewayLlmCaller.structured: model returned non-JSON for schema ${req.schemaName}: ${txt.slice(0, 200)}`);
+            }
+            const result = req.schema.safeParse(parsed);
+            if (!result.success) {
+                throw new Error(`GatewayLlmCaller.structured: schema validation failed for ${req.schemaName}: ${result.error.message}`);
+            }
+            return {
+                data: result.data,
+                usage: this.usage(raw),
+                requestId: raw.id,
+            };
+        };
+        try {
+            const out = await run(req.model);
+            return { ...out, attemptedModels: attempted, usedFallback: false };
+        }
+        catch (err) {
+            if (req.fallbackModel && err instanceof TransientLlmError) {
+                const out = await run(req.fallbackModel);
+                return { ...out, attemptedModels: attempted, usedFallback: true };
+            }
+            throw err;
+        }
+    }
+    structuredStream(req) {
+        // Streaming: fall back is NOT applied mid-stream (would confuse UI
+        // chunks). Transport retry IS applied on the initial connect; once
+        // bytes start flowing, a mid-stream drop propagates as an error and
+        // the caller (planner) handles it via its own retry loop.
+        let resolveResult;
+        let rejectResult;
+        const result = new Promise((resolve, reject) => {
+            resolveResult = resolve;
+            rejectResult = reject;
+        });
+        const self = this;
+        const tokens = {
+            [Symbol.asyncIterator]() {
+                return self.createStreamingIterator(req, resolveResult, rejectResult);
+            },
+        };
+        return { tokens, result };
+    }
+    async *createStreamingIterator(req, resolveResult, rejectResult) {
+        const jsonSchema = zod_1.z.toJSONSchema(req.schema, { target: "draft-7" });
+        const messages = await Promise.all(req.messages.map((m) => this.encodeMessage(m)));
+        const ctrl = new AbortController();
+        const t = setTimeout(() => ctrl.abort(), this.timeoutMs);
+        let accumulated = "";
+        let requestId;
+        let usage;
+        try {
+            // Connect attempt with retry on transient connect failures (pre-stream).
+            const res = await this.withRetry(() => this.fetchImpl(`${this.cfg.baseUrl.replace(/\/$/, "")}/chat/completions`, {
+                method: "POST",
+                headers: {
+                    "content-type": "application/json",
+                    authorization: `Bearer ${this.cfg.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.modelId(req.model),
+                    messages,
+                    temperature: req.temperature,
+                    max_tokens: req.maxTokens,
+                    stream: true,
+                    response_format: {
+                        type: "json_schema",
+                        json_schema: {
+                            name: req.schemaName,
+                            schema: jsonSchema,
+                            strict: true,
+                        },
+                    },
+                }),
+                signal: ctrl.signal,
+            }).then((r) => {
+                if (!r.ok && TRANSIENT_HTTP_STATUSES.has(r.status)) {
+                    throw new TransientLlmError(`GatewayLlmCaller.structuredStream: ${r.status} ${r.statusText}`);
+                }
+                return r;
+            }));
+            if (!res.ok || !res.body) {
+                const text = await res.text().catch(() => "");
+                const err = new Error(`GatewayLlmCaller.structuredStream: ${res.status} ${res.statusText} — ${text.slice(0, 500)}`);
+                rejectResult(err);
+                throw err;
+            }
+            const reader = res.body.getReader();
+            const decoder = new TextDecoder();
+            let buf = "";
+            while (true) {
+                const { value, done } = await reader.read();
+                if (done)
+                    break;
+                buf += decoder.decode(value, { stream: true });
+                let nl;
+                while ((nl = buf.indexOf("\n")) !== -1) {
+                    const line = buf.slice(0, nl).trim();
+                    buf = buf.slice(nl + 1);
+                    if (!line || !line.startsWith("data:"))
+                        continue;
+                    const payload = line.slice(5).trim();
+                    if (payload === "[DONE]")
+                        break;
+                    try {
+                        const obj = JSON.parse(payload);
+                        if (obj.id)
+                            requestId = obj.id;
+                        if (obj.usage) {
+                            usage = {
+                                promptTokens: obj.usage.prompt_tokens,
+                                completionTokens: obj.usage.completion_tokens,
+                            };
+                        }
+                        const delta = obj.choices?.[0]?.delta?.content ?? "";
+                        if (delta) {
+                            accumulated += delta;
+                            yield delta;
+                        }
+                    }
+                    catch {
+                        // skip malformed line
+                    }
+                }
+            }
+            let parsed;
+            try {
+                parsed = JSON.parse(accumulated);
+            }
+            catch {
+                const err = new Error(`GatewayLlmCaller.structuredStream: model returned non-JSON for ${req.schemaName}: ${accumulated.slice(0, 200)}`);
+                rejectResult(err);
+                throw err;
+            }
+            const validated = req.schema.safeParse(parsed);
+            if (!validated.success) {
+                const err = new Error(`GatewayLlmCaller.structuredStream: schema validation failed for ${req.schemaName}: ${validated.error.message}`);
+                rejectResult(err);
+                throw err;
+            }
+            resolveResult({
+                data: validated.data,
+                usage,
+                requestId,
+                attemptedModels: [req.model],
+                usedFallback: false,
+            });
+        }
+        catch (e) {
+            rejectResult(e);
+            throw e;
+        }
+        finally {
+            clearTimeout(t);
+        }
+    }
+    modelId(m) {
+        return `${m.provider}/${m.modelId}`;
+    }
+    async encodeMessage(m) {
+        if (!m.imageUrls?.length) {
+            return { role: m.role, content: m.content };
+        }
+        // Fetch + base64-encode each URL into a data URI. The gateway's Anthropic
+        // translator rejects raw https URLs ("URL sources are not supported"); data
+        // URIs round-trip through every provider's OpenAI-compat shim correctly.
+        const dataUris = await Promise.all(m.imageUrls.map((u) => this.toDataUri(u)));
+        return {
+            role: m.role,
+            content: [
+                { type: "text", text: m.content },
+                ...dataUris.map((url) => ({ type: "image_url", image_url: { url } })),
+            ],
+        };
+    }
+    async toDataUri(url) {
+        if (url.startsWith("data:"))
+            return url;
+        const res = await this.fetchImpl(url);
+        if (!res.ok) {
+            throw new Error(`GatewayLlmCaller.toDataUri: ${res.status} ${res.statusText} fetching ${url}`);
+        }
+        const contentType = res.headers.get("content-type")?.split(";")[0]?.trim() || "image/jpeg";
+        const buf = Buffer.from(await res.arrayBuffer());
+        return `data:${contentType};base64,${buf.toString("base64")}`;
+    }
+    usage(raw) {
+        if (!raw.usage)
+            return undefined;
+        return {
+            promptTokens: raw.usage.prompt_tokens,
+            completionTokens: raw.usage.completion_tokens,
+        };
+    }
+    /**
+     * POST /chat/completions with transient-error retry (F1).
+     * Throws TransientLlmError after exhausting retries on transient failures —
+     * caller catches it to trigger model fallback (F2). Non-transient errors
+     * (4xx other than 408/425/429, malformed responses) propagate as plain Error.
+     */
+    async postWithRetry(body) {
+        return this.withRetry(() => this.postOnce(body));
+    }
+    async withRetry(fn) {
+        let lastErr;
+        for (let attempt = 0; attempt < this.maxRetries; attempt++) {
+            try {
+                return await fn();
+            }
+            catch (err) {
+                lastErr = err;
+                const transient = err instanceof TransientLlmError || isTransientNetworkError(err);
+                if (!transient || attempt === this.maxRetries - 1) {
+                    if (transient) {
+                        // Wrap so the fallback-model path can catch it explicitly.
+                        throw err instanceof TransientLlmError
+                            ? err
+                            : new TransientLlmError(`transient LLM error after ${attempt + 1} attempts: ${String(err?.message ?? err)}`, err);
+                    }
+                    throw err;
+                }
+                await new Promise((r) => setTimeout(r, this.retryBaseMs * Math.pow(4, attempt)));
+            }
+        }
+        throw lastErr;
+    }
+    async postOnce(body) {
+        const ctrl = new AbortController();
+        const t = setTimeout(() => ctrl.abort(), this.timeoutMs);
+        try {
+            const res = await this.fetchImpl(`${this.cfg.baseUrl.replace(/\/$/, "")}/chat/completions`, {
+                method: "POST",
+                headers: {
+                    "content-type": "application/json",
+                    authorization: `Bearer ${this.cfg.apiKey}`,
+                },
+                body: JSON.stringify(body),
+                signal: ctrl.signal,
+            });
+            if (!res.ok) {
+                const text = await res.text().catch(() => "");
+                const msg = `GatewayLlmCaller: ${res.status} ${res.statusText} — ${text.slice(0, 500)}`;
+                if (TRANSIENT_HTTP_STATUSES.has(res.status)) {
+                    throw new TransientLlmError(msg);
+                }
+                throw new Error(msg);
+            }
+            return (await res.json());
+        }
+        finally {
+            clearTimeout(t);
+        }
+    }
+}
+exports.GatewayLlmCaller = GatewayLlmCaller;

package/lib/services/agent/llmCallerRegistry.d.ts

@@ -0,0 +1,6 @@
+import { LlmCaller } from "./llmCaller";
+export declare function setLlmCaller(caller: LlmCaller): void;
+export declare function getLlmCaller(): LlmCaller;
+export declare function setAnthropicCaller(caller: LlmCaller | null): void;
+export declare function getAnthropicCaller(): LlmCaller | null;
+//# sourceMappingURL=llmCallerRegistry.d.ts.map

package/lib/services/agent/llmCallerRegistry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llmCallerRegistry.d.ts","sourceRoot":"","sources":["../../../src/services/agent/llmCallerRegistry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAyB,MAAM,aAAa,CAAC;AAY/D,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAEpD;AAED,wBAAgB,YAAY,IAAI,SAAS,CAExC;AAeD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,GAAG,IAAI,CAEjE;AAED,wBAAgB,kBAAkB,IAAI,SAAS,GAAG,IAAI,CAErD"}

package/lib/services/agent/llmCallerRegistry.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setLlmCaller = setLlmCaller;
+exports.getLlmCaller = getLlmCaller;
+exports.setAnthropicCaller = setAnthropicCaller;
+exports.getAnthropicCaller = getAnthropicCaller;
+const llmCaller_1 = require("./llmCaller");
+/**
+ * Singleton LlmCaller accessor. Host wires a concrete impl at boot
+ * (e.g. GatewayLlmCaller); agent code reads via getLlmCaller().
+ *
+ * Throwing UnconfiguredLlmCaller is the default so tests fail loud
+ * if the host forgets to wire.
+ */
+let instance = new llmCaller_1.UnconfiguredLlmCaller();
+function setLlmCaller(caller) {
+    instance = caller;
+}
+function getLlmCaller() {
+    return instance;
+}
+/**
+ * Optional Anthropic-direct caller. Wired only on hosts that have
+ * ANTHROPIC_API_KEY set and need Anthropic-specific features (server-side
+ * web_search, prompt caching, extended thinking). The Planner reads this
+ * only when `useWebResearch` is requested — otherwise the default gateway
+ * caller is used so provider portability is preserved.
+ *
+ * Null when unconfigured rather than throwing — caller code checks and
+ * gracefully falls back to the gateway path. See AnthropicLlmCaller docstring
+ * for deferred features (Tavily custom tools, fallback chain, streaming).
+ */
+let anthropicInstance = null;
+function setAnthropicCaller(caller) {
+    anthropicInstance = caller;
+}
+function getAnthropicCaller() {
+    return anthropicInstance;
+}

package/lib/services/agent/modelQualityNotes.d.ts

@@ -0,0 +1,100 @@
+/**
+ * AG-39: per-model capability scorecard fed by AG-26 critic findings.
+ *
+ * Why: planner picks `preferredModel` heuristically (tier + cost). It has
+ * no evidence about which models actually deliver on capabilities that
+ * matter for a given scene — brand-text legibility, character
+ * continuity, fast-motion fidelity, etc. After the first time
+ * seedance-1.0-pro-fast mangled the "HydraPure" label in test 1 we needed
+ * a way to (a) tell the planner "avoid X for scenes needing Y" and (b)
+ * automatically degrade trust in a model as critic evidence accumulates,
+ * without manually maintaining a blacklist.
+ *
+ * Threshold (so a single bad scene doesn't kill a model):
+ *   - score decays by 1 ONLY when both:
+ *     * >=3 distinct FAIL findings for that capability in the last 10
+ *       attempts (recent window), AND
+ *     * failure rate within that window >= 0.30
+ *   - scores are clamped to [1, 5]
+ *   - a PASS finding in the window adds back attemptCount only; no
+ *     auto-recovery of score (we'd rather a human re-rate a model than
+ *     trust ourselves to undo a downgrade).
+ *
+ * Storage: a small JSON file under
+ *   wt-dev-backend/packages/shared/data/model-quality-notes.json
+ * checked in to the repo. Reads are memoised per-process; writes are
+ * append-only with an exclusive lock so the critic feedback hook can run
+ * inline with the agent run without racing.
+ *
+ * Surface for the planner: `renderModelQualityHintsForSystemPrompt()`
+ * returns a short paragraph listing models to AVOID per capability,
+ * filtered to only what the planner needs to know (no full dump — keeps
+ * the system prompt small).
+ */
+export type Capability = "textRendering" | "characterContinuity" | "fastMotion" | "photoreal" | "productFidelity" | "cinematic" | "audioSync";
+export interface QualityEvidence {
+    /** ISO timestamp the evidence was recorded. */
+    at: string;
+    /** Critic's grade for this capability on this scene. */
+    verdict: "pass" | "warn" | "fail";
+    /** Short why (from the critic's `message`). */
+    message: string;
+    /** Agent run + scene the evidence came from (for audit). */
+    runId: string;
+    sceneIndex?: number;
+}
+export interface ModelCapabilityEntry {
+    /** 1-5; 5 = best in the catalog at this capability. Hand-seeded, decayed by critic. */
+    score: number;
+    /** Why the seed score was picked, plus any human notes. */
+    seedNote?: string;
+    /** Recent evidence — rolling window of the last 10 entries. */
+    evidence: QualityEvidence[];
+}
+export interface ModelQualityEntry {
+    modelKey: string;
+    capabilities: Partial<Record<Capability, ModelCapabilityEntry>>;
+    lastUpdated: string;
+}
+export declare function getModelCapabilityScore(modelKey: string, capability: Capability): number | undefined;
+/**
+ * Return a short paragraph the planner system prompt can paste in,
+ * listing models to AVOID per capability. Only emits hints for
+ * capabilities we have rated AND that the brief signals are relevant
+ * (so the planner isn't drowned in noise).
+ */
+export declare function renderModelQualityHintsForSystemPrompt(opts: {
+    needsBrandText?: boolean;
+    needsCharacterContinuity?: boolean;
+    needsFastMotion?: boolean;
+    needsCinematic?: boolean;
+}): string;
+/**
+ * Critic feedback hook: called after every stitched-video critic run
+ * with the structured findings. We map each finding's `dimension` to a
+ * `Capability`, record evidence under the model that rendered the scene,
+ * and decay the score iff the recent-window thresholds are met.
+ */
+export declare function recordCriticEvidence(input: {
+    runId: string;
+    /** sceneIndex → modelKey that was actually used (from trace.outcomes). */
+    sceneModels: Map<number, string>;
+    findings: Array<{
+        severity: "pass" | "warn" | "fail";
+        dimension: string;
+        message: string;
+        sceneIndexHint?: number;
+    }>;
+}): {
+    decays: Array<{
+        modelKey: string;
+        capability: Capability;
+        from: number;
+        to: number;
+    }>;
+};
+/** For tests + manual inspection. */
+export declare function _dumpStore(): ModelQualityEntry[];
+/** For tests — reset to seed (does NOT touch disk). */
+export declare function _resetCacheForTests(): void;
+//# sourceMappingURL=modelQualityNotes.d.ts.map

package/lib/services/agent/modelQualityNotes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"modelQualityNotes.d.ts","sourceRoot":"","sources":["../../../src/services/agent/modelQualityNotes.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,MAAM,MAAM,UAAU,GAClB,eAAe,GACf,qBAAqB,GACrB,YAAY,GACZ,WAAW,GACX,iBAAiB,GACjB,WAAW,GACX,WAAW,CAAC;AAEhB,MAAM,WAAW,eAAe;IAC9B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,wDAAwD;IACxD,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IAClC,+CAA+C;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACnC,uFAAuF;IACvF,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+DAA+D;IAC/D,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC,CAAC;IAChE,WAAW,EAAE,MAAM,CAAC;CACrB;AAmGD,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,UAAU,GACrB,MAAM,GAAG,SAAS,CAGpB;AAED;;;;;GAKG;AACH,wBAAgB,sCAAsC,CAAC,IAAI,EAAE;IAC3D,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,MAAM,CA2BT;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,0EAA0E;IAC1E,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,KAAK,CAAC;QACd,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;QACnC,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC,CAAC;CACJ,GAAG;IAAE,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAAE,CAiE5F;AAED,qCAAqC;AACrC,wBAAgB,UAAU,IAAI,iBAAiB,EAAE,CAEhD;AAED,uDAAuD;AACvD,wBAAgB,mBAAmB,SAElC"}