waypoi 0.0.0

package/dist/src/services/imageUnderstanding.js

@@ -0,0 +1,352 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runImageUnderstanding = runImageUnderstanding;
+exports.resolveImageInputToUrl = resolveImageInputToUrl;
+exports.resolveImageInput = resolveImageInput;
+exports.imageDataUrlFromPath = imageDataUrlFromPath;
+exports.imageDataUrlWithGeometryFromPath = imageDataUrlWithGeometryFromPath;
+exports.buildImageGeometrySystemMessage = buildImageGeometrySystemMessage;
+exports.parseImageUnderstandingText = parseImageUnderstandingText;
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const sharp_1 = __importDefault(require("sharp"));
+const router_1 = require("../routing/router");
+const scheduler_1 = require("../pools/scheduler");
+const modelRegistry_1 = require("../providers/modelRegistry");
+const DEFAULT_INSTRUCTION = "Analyze this image. Return OCR text, key objects, scene summary, and notable details.";
+const MAX_IMAGE_PIXELS = 1080 * 720 - 1;
+const RESIZE_QUALITY = 85;
+async function runImageUnderstanding(paths, input, signal) {
+    const model = await resolveVisionTextModel(paths, input.model);
+    if (!model) {
+        throw typedError("no_vision_model", "No vision-capable text model available.");
+    }
+    const resolvedImage = await resolveImageInput(input);
+    const instruction = input.instruction?.trim() ? input.instruction : DEFAULT_INSTRUCTION;
+    const messages = [];
+    if (resolvedImage.imageGeometry) {
+        messages.push({
+            role: "system",
+            content: buildImageGeometrySystemMessage(resolvedImage.imageGeometry),
+        });
+    }
+    messages.push({
+        role: "user",
+        content: [
+            { type: "image_url", image_url: { url: resolvedImage.imageUrl } },
+            { type: "text", text: instruction },
+        ],
+    });
+    const payload = {
+        model,
+        stream: false,
+        messages,
+    };
+    if (typeof input.max_tokens === "number") {
+        payload.max_tokens = input.max_tokens;
+    }
+    if (typeof input.temperature === "number") {
+        payload.temperature = input.temperature;
+    }
+    const outcome = await (0, router_1.routeRequest)(paths, model, "/v1/chat/completions", payload, {}, signal, {
+        requiredInput: ["text", "image"],
+        requiredOutput: ["text"],
+    });
+    const responsePayload = await readBody(outcome.attempt.response);
+    const rawText = extractAssistantText(responsePayload.payload);
+    const analysis = parseImageUnderstandingText(rawText);
+    return {
+        model,
+        analysis,
+        raw_text: rawText,
+        image_geometry: resolvedImage.imageGeometry,
+        usage: {
+            prompt_tokens: responsePayload.usage?.prompt_tokens ?? 0,
+            completion_tokens: responsePayload.usage?.completion_tokens ?? 0,
+            total_tokens: responsePayload.usage?.total_tokens ?? 0,
+        },
+    };
+}
+async function resolveImageInputToUrl(input) {
+    const resolved = await resolveImageInput(input);
+    return resolved.imageUrl;
+}
+async function resolveImageInput(input) {
+    if (input.image_path) {
+        return imageDataUrlWithGeometryFromPath(input.image_path);
+    }
+    if (input.image_url && isValidImageUrl(input.image_url)) {
+        return { imageUrl: input.image_url };
+    }
+    throw typedError("invalid_request", "Exactly one image source is required: image_path or image_url.");
+}
+async function imageDataUrlFromPath(imagePath) {
+    const resolved = await imageDataUrlWithGeometryFromPath(imagePath);
+    return resolved.imageUrl;
+}
+async function imageDataUrlWithGeometryFromPath(imagePath) {
+    const abs = path_1.default.resolve(imagePath);
+    let data;
+    try {
+        data = await fs_1.promises.readFile(abs);
+    }
+    catch {
+        throw typedError("invalid_request", `image_path not readable: ${imagePath}`);
+    }
+    let mimeType = mimeFromExt(abs);
+    const image = (0, sharp_1.default)(data);
+    const meta = await image.metadata();
+    if (!meta.width || !meta.height) {
+        throw typedError("invalid_request", "Unable to read image dimensions.");
+    }
+    const originalWidth = meta.width;
+    const originalHeight = meta.height;
+    let uploadedWidth = originalWidth;
+    let uploadedHeight = originalHeight;
+    const area = meta.width * meta.height;
+    if (area > MAX_IMAGE_PIXELS) {
+        const scale = Math.sqrt(MAX_IMAGE_PIXELS / area);
+        const targetWidth = Math.max(1, Math.floor(meta.width * scale));
+        const targetHeight = Math.max(1, Math.floor(meta.height * scale));
+        uploadedWidth = targetWidth;
+        uploadedHeight = targetHeight;
+        let resized = image.resize(targetWidth, targetHeight, { fit: "fill" });
+        const format = (meta.format ?? "").toLowerCase();
+        if (format === "jpeg" || format === "jpg") {
+            resized = resized.jpeg({ quality: RESIZE_QUALITY });
+            mimeType = "image/jpeg";
+        }
+        else if (format === "png") {
+            resized = resized.png();
+            mimeType = "image/png";
+        }
+        else if (format === "webp") {
+            resized = resized.webp({ quality: RESIZE_QUALITY });
+            mimeType = "image/webp";
+        }
+        else {
+            resized = resized.png();
+            mimeType = "image/png";
+        }
+        data = await resized.toBuffer();
+    }
+    return {
+        imageUrl: `data:${mimeType};base64,${data.toString("base64")}`,
+        imageGeometry: {
+            original_width: originalWidth,
+            original_height: originalHeight,
+            uploaded_width: uploadedWidth,
+            uploaded_height: uploadedHeight,
+            scale_x: originalWidth / uploadedWidth,
+            scale_y: originalHeight / uploadedHeight,
+            resized: originalWidth !== uploadedWidth || originalHeight !== uploadedHeight,
+        },
+    };
+}
+function buildImageGeometrySystemMessage(imageGeometry) {
+    return [
+        "If you return coordinates or bounding boxes, express them in the original image pixel space.",
+        `Original image size: ${imageGeometry.original_width}x${imageGeometry.original_height}.`,
+        `Uploaded image size: ${imageGeometry.uploaded_width}x${imageGeometry.uploaded_height}.`,
+        `Scale factors from uploaded to original: x=${imageGeometry.scale_x}, y=${imageGeometry.scale_y}.`,
+        "Do not return coordinates in resized-image pixels.",
+    ].join(" ");
+}
+function parseImageUnderstandingText(rawText) {
+    const trimmed = rawText.trim();
+    const parsedJson = parseEmbeddedJson(trimmed);
+    if (parsedJson) {
+        return fromJsonAnalysis(parsedJson, trimmed);
+    }
+    const answer = trimmed || "No textual response returned.";
+    const lines = answer.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+    return {
+        answer,
+        ocr_text: extractKeyValue(lines, "ocr_text") ?? extractKeyValue(lines, "ocr") ?? "",
+        objects: splitList(extractKeyValue(lines, "objects") ?? ""),
+        scene: extractKeyValue(lines, "scene") ?? "",
+        notable_details: splitList(extractKeyValue(lines, "notable_details") ?? extractKeyValue(lines, "notable details") ?? ""),
+        safety_notes: splitList(extractKeyValue(lines, "safety_notes") ?? extractKeyValue(lines, "safety notes") ?? ""),
+    };
+}
+async function resolveVisionTextModel(paths, requestedModel) {
+    if (requestedModel) {
+        return requestedModel;
+    }
+    const smart = await (0, scheduler_1.selectPoolCandidates)(paths, "smart", {
+        requiredInput: ["text", "image"],
+        requiredOutput: ["text"],
+    }, {
+        operation: "chat_completions",
+        stream: false,
+    });
+    if (smart && smart.candidates.length > 0) {
+        return "smart";
+    }
+    return (0, modelRegistry_1.pickBestProviderModelByCapabilities)(paths, { requiredInput: ["text", "image"], requiredOutput: ["text"] }, "llm");
+}
+async function readBody(response) {
+    const chunks = [];
+    for await (const chunk of response.body) {
+        chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    }
+    const buffer = Buffer.concat(chunks);
+    const contentType = normalizeContentType(response.headers);
+    const rawText = buffer.toString("utf8");
+    let payload;
+    try {
+        payload = JSON.parse(rawText);
+    }
+    catch {
+        const status = typeof response.statusCode === "number" ? response.statusCode : 0;
+        const snippet = summarizeBodySnippet(rawText);
+        throw typedError("upstream_error", `Expected JSON from chat completion. status=${status} content-type=${contentType || "unknown"} body=${snippet}`);
+    }
+    return { payload, usage: payload.usage ?? null };
+}
+function extractAssistantText(payload) {
+    if (!payload || typeof payload !== "object") {
+        return "";
+    }
+    const choices = payload.choices;
+    if (!Array.isArray(choices) || choices.length === 0) {
+        return "";
+    }
+    const first = choices[0];
+    const content = first?.message?.content;
+    if (typeof content === "string") {
+        return content;
+    }
+    if (Array.isArray(content)) {
+        const texts = content
+            .map((part) => {
+            if (!part || typeof part !== "object")
+                return "";
+            const type = part.type;
+            if (type !== "text")
+                return "";
+            return part.text ?? "";
+        })
+            .filter(Boolean);
+        return texts.join("\n").trim();
+    }
+    return "";
+}
+function parseEmbeddedJson(text) {
+    const fencedMatch = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
+    const candidate = fencedMatch ? fencedMatch[1] : text;
+    try {
+        const parsed = JSON.parse(candidate);
+        return parsed && typeof parsed === "object" ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+function fromJsonAnalysis(json, fallbackRaw) {
+    const analysis = (json.analysis && typeof json.analysis === "object"
+        ? json.analysis
+        : json);
+    const answer = asString(analysis.answer) ??
+        asString(json.answer) ??
+        asString(json.raw_text) ??
+        asString(fallbackRaw) ??
+        "No textual response returned.";
+    return {
+        answer,
+        ocr_text: asString(analysis.ocr_text) ?? "",
+        objects: asStringArray(analysis.objects),
+        scene: asString(analysis.scene) ?? "",
+        notable_details: asStringArray(analysis.notable_details),
+        safety_notes: asStringArray(analysis.safety_notes),
+    };
+}
+function asString(value) {
+    if (typeof value === "string") {
+        const trimmed = value.trim();
+        return trimmed.length > 0 ? trimmed : null;
+    }
+    return null;
+}
+function asStringArray(value) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    return value
+        .map((entry) => (typeof entry === "string" ? entry.trim() : ""))
+        .filter(Boolean);
+}
+function splitList(value) {
+    if (!value)
+        return [];
+    return value
+        .split(/[,\n]/)
+        .map((item) => item.replace(/^[-*]\s*/, "").trim())
+        .filter(Boolean);
+}
+function extractKeyValue(lines, key) {
+    const pattern = new RegExp(`^${escapeRegExp(key)}\\s*:\\s*(.+)$`, "i");
+    for (const line of lines) {
+        const match = line.match(pattern);
+        if (match) {
+            return match[1].trim();
+        }
+    }
+    return null;
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function isValidImageUrl(url) {
+    if (url.startsWith("data:")) {
+        return /^data:[^;]+;base64,/i.test(url);
+    }
+    try {
+        const parsed = new URL(url);
+        return parsed.protocol === "http:" || parsed.protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+}
+function normalizeContentType(headers) {
+    const ct = headers["content-type"] ?? headers["Content-Type"];
+    if (Array.isArray(ct))
+        return ct.join(", ");
+    return ct ?? "";
+}
+function summarizeBodySnippet(body) {
+    const trimmed = body.replace(/\s+/g, " ").trim();
+    if (!trimmed) {
+        return "<empty>";
+    }
+    const max = 1024;
+    if (trimmed.length <= max) {
+        return trimmed;
+    }
+    return `${trimmed.slice(0, max)}…`;
+}
+function mimeFromExt(filePath) {
+    const ext = path_1.default.extname(filePath).slice(1).toLowerCase();
+    const map = {
+        png: "image/png",
+        jpg: "image/jpeg",
+        jpeg: "image/jpeg",
+        gif: "image/gif",
+        webp: "image/webp",
+        bmp: "image/bmp",
+        svg: "image/svg+xml",
+        tif: "image/tiff",
+        tiff: "image/tiff",
+    };
+    return map[ext] ?? "application/octet-stream";
+}
+function typedError(type, message) {
+    const error = new Error(message);
+    error.type = type;
+    error.retryable = false;
+    return error;
+}

package/dist/src/services/videoGeneration.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveVideoGenerationModel = resolveVideoGenerationModel;
+exports.runVideoGeneration = runVideoGeneration;
+const router_1 = require("../routing/router");
+const scheduler_1 = require("../pools/scheduler");
+const modelRegistry_1 = require("../providers/modelRegistry");
+async function resolveVideoGenerationModel(paths, requestedModel) {
+    if (requestedModel) {
+        return requestedModel;
+    }
+    return pickDefaultVideoModel(paths);
+}
+async function runVideoGeneration(paths, request, headers, signal) {
+    const model = await resolveVideoGenerationModel(paths, request.model);
+    if (!model) {
+        const error = new Error("No video generation model available. Add or enable a provider model.");
+        error.type = "no_video_model";
+        error.retryable = false;
+        throw error;
+    }
+    const outcome = await (0, router_1.routeRequest)(paths, model, "/v1/videos/generations", { ...request, model }, headers, signal, {
+        endpointType: "video",
+        requiredInput: ["text"],
+        requiredOutput: ["video"],
+    });
+    const body = await readBody(outcome.attempt.response);
+    return {
+        model,
+        statusCode: outcome.attempt.response.statusCode,
+        headers: outcome.attempt.response.headers,
+        payload: body.payload,
+        route: {
+            endpointId: outcome.attempt.endpoint.id,
+            endpointName: outcome.attempt.endpoint.name,
+            upstreamModel: outcome.attempt.upstreamModel,
+        },
+    };
+}
+async function pickDefaultVideoModel(paths) {
+    const smart = await (0, scheduler_1.selectPoolCandidates)(paths, "smart", {
+        requiredInput: ["text"],
+        requiredOutput: ["video"],
+    }, {
+        operation: "video_generations",
+        stream: false,
+    });
+    if (smart && smart.candidates.length > 0) {
+        return "smart";
+    }
+    const byCapabilities = await (0, modelRegistry_1.pickBestProviderModelByCapabilities)(paths, { requiredInput: ["text"], requiredOutput: ["video"] }, "video");
+    if (byCapabilities) {
+        return byCapabilities;
+    }
+    return null;
+}
+async function readBody(response) {
+    const chunks = [];
+    for await (const chunk of response.body) {
+        chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    }
+    const buffer = Buffer.concat(chunks);
+    const contentType = normalizeContentType(response.headers);
+    if (contentType.includes("application/json")) {
+        try {
+            return { payload: JSON.parse(buffer.toString("utf8")) };
+        }
+        catch {
+            return { payload: buffer };
+        }
+    }
+    return { payload: buffer };
+}
+function normalizeContentType(headers) {
+    const ct = headers["content-type"] ?? headers["Content-Type"];
+    if (Array.isArray(ct))
+        return ct.join(", ");
+    return ct ?? "";
+}