vidspotai-shared 1.0.67 → 1.0.69-dev.0

package/lib/services/agent/tools/animateImageWithMotionBrush.tool.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.animateImageWithMotionBrushTool = void 0;
+const zod_1 = require("zod");
+const aiModels_1 = require("../../../globals/aiModels");
+const logger_1 = require("../../../utils/logger");
+const aiGenFactory_service_1 = require("../../aiGen/aiGenFactory.service");
+const toolRegistry_1 = require("../toolRegistry");
+const chains_1 = require("../providerFallback/chains");
+const withFallback_1 = require("../providerFallback/withFallback");
+// Motion-brush is Kling-only and only on v3 + v2.6 per Kling's capability map.
+// v2.5-turbo and v2.1 reject mask payloads, so the chain stays within the
+// motion-brush-capable family on failure.
+const MOTION_BRUSH_CHAIN = [
+    aiModels_1.EVideoGenModels.KLING_V3_0,
+    aiModels_1.EVideoGenModels.KLING_V2_6,
+];
+const TrajectoryPoint = zod_1.z.object({
+    x: zod_1.z.number().describe("X in pixels of the input image's native resolution."),
+    y: zod_1.z.number().describe("Y in pixels of the input image's native resolution."),
+});
+const DynamicMask = zod_1.z.object({
+    maskUrl: zod_1.z
+        .string()
+        .url()
+        .describe("Public PNG URL. Same dimensions as inputImageUrl. Opaque pixels = the region that should follow the trajectory."),
+    trajectories: zod_1.z
+        .array(TrajectoryPoint)
+        .min(2)
+        .describe("≥2 polyline points (native pixel coords). Interpreted as motion path over the clip duration."),
+});
+const InputSchema = zod_1.z.object({
+    imageUrl: zod_1.z
+        .string()
+        .url()
+        .describe("Source still — the image the masks were painted against."),
+    motionHint: zod_1.z
+        .string()
+        .min(1)
+        .describe("Plain-language motion description for the unmasked / dynamic regions, e.g. \"clouds drift right while the person stays still\"."),
+    staticMaskUrl: zod_1.z
+        .string()
+        .url()
+        .optional()
+        .describe("Optional PNG mask URL. Same dimensions as imageUrl. Opaque = keep frozen."),
+    dynamicMasks: zod_1.z
+        .array(DynamicMask)
+        .max(6)
+        .optional()
+        .describe("Up to 6 motion-brush regions. Each has a mask PNG + a trajectory polyline."),
+    modelKey: zod_1.z
+        .nativeEnum(aiModels_1.EVideoGenModels)
+        .default(aiModels_1.EVideoGenModels.KLING_V2_6)
+        .describe("Kling I2V model that supports motion-brush (v3.0 or v2.6 only)."),
+    durationSec: zod_1.z.number().int().min(2).max(10).default(5),
+    aspectRatio: zod_1.z.enum(["9:16", "16:9", "1:1"]).default("9:16"),
+});
+const OutputSchema = zod_1.z.object({
+    status: zod_1.z.literal("triggered"),
+    taskId: zod_1.z.string(),
+    raw: zod_1.z.unknown(),
+    modelKeyUsed: zod_1.z.string().optional(),
+    attemptedProviders: zod_1.z.array(zod_1.z.string()).optional(),
+});
+exports.animateImageWithMotionBrushTool = {
+    name: "animate_image_with_motion_brush",
+    description: [
+        "Animate a still image with fine-grained Kling motion-brush control:",
+        "freeze regions with `staticMaskUrl`, and/or move regions along a path",
+        "with `dynamicMasks[]`. Use this instead of `animate_image` when the user",
+        "has already painted masks (e.g. via the motion-brush Studio Tool or the",
+        "Generate-new-video Advanced section), or when only some parts of the",
+        "image should move.",
+        "Coordinate space: trajectory points are in the input image's native",
+        "pixel resolution. Mask PNGs must match those native dimensions exactly.",
+        "Examples:",
+        '  { imageUrl, motionHint: "drift the clouds right", dynamicMasks: [{maskUrl, trajectories:[{x:120,y:80},{x:540,y:80}]}] }',
+        '  { imageUrl, motionHint: "person nods", staticMaskUrl: "https://…background.png", dynamicMasks: [{maskUrl, trajectories:[{x:300,y:400},{x:300,y:380}]}] }',
+    ].join(" "),
+    inputSchema: InputSchema,
+    outputSchema: OutputSchema,
+    idempotent: false,
+    estimateCost: async (input) => Math.ceil(input.durationSec * 5),
+    handler: async (input) => {
+        const chain = (0, chains_1.buildVideoChain)(input.modelKey, MOTION_BRUSH_CHAIN);
+        const outcome = await (0, withFallback_1.withProviderFallback)({
+            providers: chain,
+            capable: (m) => (0, chains_1.isVideoModelCapable)(m, {
+                aspect: input.aspectRatio,
+                durationSec: input.durationSec,
+                type: "image-to-video",
+                hasInputImage: true,
+            }),
+            onAttempt: (a) => {
+                if (a.classification) {
+                    logger_1.logger.warn("animate_image_motion_brush provider attempt failed", {
+                        provider: String(a.provider),
+                        classification: a.classification.classification,
+                        needsReplan: a.classification.needsReplan,
+                        err: a.classification.message,
+                        durationMs: a.durationMs,
+                    });
+                }
+            },
+            call: async (modelKey) => {
+                const provider = (0, aiGenFactory_service_1.getAiGenProviderService)(modelKey);
+                const result = await provider.generateVideo({
+                    prompt: input.motionHint,
+                    modelKey,
+                    duration: input.durationSec,
+                    aspectRatio: input.aspectRatio,
+                    inputImageUrl: input.imageUrl,
+                    type: "image-to-video",
+                    staticMaskUrl: input.staticMaskUrl,
+                    dynamicMasks: input.dynamicMasks,
+                });
+                const taskId = result.task?.taskId ??
+                    result.task?.id ??
+                    (typeof result.task === "string" ? result.task : "unknown");
+                return { taskId, raw: result.task, modelKey };
+            },
+        });
+        if (!outcome.ok) {
+            throw new withFallback_1.ProviderFallbackExhaustedError(outcome.classification.classification, outcome.needsReplan, outcome.attempts.map((a) => String(a.provider)), `animate_image_with_motion_brush: chain exhausted (${outcome.classification.classification}): ${outcome.classification.message}`);
+        }
+        return {
+            status: "triggered",
+            taskId: outcome.value.taskId,
+            raw: outcome.value.raw,
+            modelKeyUsed: outcome.value.modelKey,
+            attemptedProviders: outcome.attempts.map((a) => String(a.provider)),
+        };
+    },
+};
+(0, toolRegistry_1.registerTool)(exports.animateImageWithMotionBrushTool);