waypoi 0.0.0

package/dist/src/protocols/adapters/inferenceV2.js

@@ -0,0 +1,346 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inferenceV2ProtocolAdapter = void 0;
+const fs_1 = require("fs");
+const stream_1 = require("stream");
+const imageCache_1 = require("../../storage/imageCache");
+const DEFAULT_RESPONSE_TEXT_PATHS = [
+    "outputs.0.outputs.text",
+    "outputs.0.text",
+    "outputs.0.generated_text",
+];
+exports.inferenceV2ProtocolAdapter = {
+    id: "inference_v2",
+    supportedOperations: ["chat_completions"],
+    streamSupportedOperations: [],
+    supports(context) {
+        if (context.operation !== "chat_completions") {
+            return { supported: false, reason: "unsupported_operation" };
+        }
+        if (context.stream) {
+            return { supported: false, reason: "stream_unsupported" };
+        }
+        return { supported: true };
+    },
+    async buildRequest(context) {
+        const router = typeof context.config?.router === "string" ? context.config.router.trim() : "";
+        if (!router) {
+            throw protocolError("invalid_protocol_config", "inference_v2 protocol requires provider protocolConfig.router.", false);
+        }
+        const { text, image } = await extractPromptAndImage(context.paths, context.payload);
+        const inferInputs = {
+            text,
+            max_new_tokens: numberOrUndefined(context.payload.max_tokens),
+            temperature: numberOrUndefined(context.payload.temperature),
+            top_p: numberOrUndefined(context.payload.top_p),
+        };
+        if (image) {
+            inferInputs.image = image;
+        }
+        const payload = {
+            inputs: [
+                {
+                    model_name: context.upstreamModel,
+                    inputs: compactObject(inferInputs),
+                },
+            ],
+        };
+        const requestPath = `/v2/models/${encodeURIComponent(router)}/infer`;
+        const auth = buildAuth(context, requestPath);
+        return {
+            path: auth.path,
+            payload,
+            headers: auth.headers,
+            skipDefaultAuth: auth.skipDefaultAuth,
+        };
+    },
+    async normalizeResponse(context) {
+        const responseJson = await readJsonBody(context.upstreamResult);
+        const textPaths = normalizeTextPaths(context.config?.responseTextPaths);
+        const content = firstStringByPaths(responseJson, textPaths);
+        if (!content) {
+            throw protocolError("invalid_upstream_response", "Inference v2 response does not contain assistant text at configured paths.", true);
+        }
+        const promptTokens = numberFromPath(responseJson, "usage.prompt_tokens");
+        const completionTokens = numberFromPath(responseJson, "usage.completion_tokens");
+        const totalTokensCandidate = numberFromPath(responseJson, "usage.total_tokens");
+        const totalTokens = totalTokensCandidate ??
+            (promptTokens !== undefined && completionTokens !== undefined
+                ? promptTokens + completionTokens
+                : undefined);
+        const normalized = {
+            id: `chatcmpl-infer-${Date.now().toString(36)}`,
+            object: "chat.completion",
+            created: Math.floor(Date.now() / 1000),
+            model: context.publicModel,
+            choices: [
+                {
+                    index: 0,
+                    message: {
+                        role: "assistant",
+                        content,
+                    },
+                    finish_reason: "stop",
+                },
+            ],
+            usage: promptTokens !== undefined || completionTokens !== undefined || totalTokens !== undefined
+                ? {
+                    prompt_tokens: promptTokens ?? 0,
+                    completion_tokens: completionTokens ?? 0,
+                    total_tokens: totalTokens ?? 0,
+                }
+                : undefined,
+            waypoi_adapter: {
+                protocol: "inference_v2",
+            },
+        };
+        const buffer = Buffer.from(JSON.stringify(normalized), "utf8");
+        const headers = {
+            ...context.upstreamResult.headers,
+            "content-type": "application/json",
+        };
+        return {
+            statusCode: context.upstreamResult.statusCode,
+            headers,
+            body: stream_1.Readable.from([buffer]),
+            rawBody: buffer,
+        };
+    },
+};
+async function extractPromptAndImage(paths, payload) {
+    const messages = Array.isArray(payload.messages)
+        ? payload.messages
+        : [];
+    const message = [...messages]
+        .reverse()
+        .find((item) => item &&
+        typeof item === "object" &&
+        item.role === "user") ?? messages[messages.length - 1];
+    if (!message || typeof message !== "object") {
+        return { text: "" };
+    }
+    const content = message.content;
+    if (typeof content === "string") {
+        return { text: content };
+    }
+    if (!Array.isArray(content)) {
+        return { text: "" };
+    }
+    const textParts = [];
+    let image;
+    for (const part of content) {
+        if (!part || typeof part !== "object") {
+            continue;
+        }
+        const typed = part;
+        const type = typeof typed.type === "string" ? typed.type : "";
+        if (type === "text" || type === "input_text" || type === "output_text") {
+            const text = typed.text;
+            if (typeof text === "string" && text.length > 0) {
+                textParts.push(text);
+            }
+            continue;
+        }
+        if (!image && type === "image_url") {
+            const imageUrl = typed.image_url;
+            if (typeof imageUrl?.url === "string") {
+                image = await resolveImageToBase64(paths, imageUrl.url);
+            }
+            continue;
+        }
+        if (!image && type === "image") {
+            const value = typed.image;
+            if (typeof value === "string") {
+                image = await resolveImageToBase64(paths, value);
+            }
+            continue;
+        }
+        if (!image && type === "input_image") {
+            const imageValue = typed.image ?? typed.image_url?.url;
+            if (typeof imageValue === "string") {
+                image = await resolveImageToBase64(paths, imageValue);
+            }
+        }
+    }
+    return {
+        text: textParts.join("\n").trim(),
+        image,
+    };
+}
+async function resolveImageToBase64(paths, value) {
+    if (value.startsWith("data:")) {
+        const match = value.match(/^data:[^;]+;base64,(.+)$/i);
+        if (!match) {
+            throw protocolError("invalid_request", "Invalid data URL for image input.", false);
+        }
+        return match[1].replace(/\s+/g, "");
+    }
+    if (looksLikeBase64(value)) {
+        return value.replace(/\s+/g, "");
+    }
+    const hash = extractLocalMediaHash(value);
+    if (!hash) {
+        throw protocolError("invalid_request", "Only local /admin/media or /admin/images URLs are allowed for image input.", false);
+    }
+    const mediaPath = await (0, imageCache_1.getMediaPath)(paths, hash);
+    if (!mediaPath) {
+        throw protocolError("invalid_request", "Referenced image not found in cache.", false);
+    }
+    const buffer = await fs_1.promises.readFile(mediaPath);
+    return buffer.toString("base64");
+}
+function normalizeTextPaths(raw) {
+    if (Array.isArray(raw)) {
+        const values = raw.filter((value) => typeof value === "string" && value.length > 0);
+        if (values.length > 0) {
+            return values;
+        }
+    }
+    return DEFAULT_RESPONSE_TEXT_PATHS;
+}
+function firstStringByPaths(source, paths) {
+    for (const path of paths) {
+        const value = getByPath(source, path);
+        if (typeof value === "string" && value.trim().length > 0) {
+            return value.trim();
+        }
+    }
+    return undefined;
+}
+function numberFromPath(source, path) {
+    const value = getByPath(source, path);
+    if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+    }
+    return undefined;
+}
+function getByPath(source, path) {
+    const segments = path
+        .replace(/\[(\d+)\]/g, ".$1")
+        .split(".")
+        .map((segment) => segment.trim())
+        .filter((segment) => segment.length > 0);
+    let cursor = source;
+    for (const segment of segments) {
+        if (cursor === null || cursor === undefined) {
+            return undefined;
+        }
+        if (Array.isArray(cursor)) {
+            const index = Number(segment);
+            if (!Number.isInteger(index) || index < 0 || index >= cursor.length) {
+                return undefined;
+            }
+            cursor = cursor[index];
+            continue;
+        }
+        if (typeof cursor !== "object") {
+            return undefined;
+        }
+        cursor = cursor[segment];
+    }
+    return cursor;
+}
+function compactObject(input) {
+    const output = {};
+    for (const [key, value] of Object.entries(input)) {
+        if (value !== undefined) {
+            output[key] = value;
+        }
+    }
+    return output;
+}
+function numberOrUndefined(value) {
+    if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+    }
+    return undefined;
+}
+function looksLikeBase64(value) {
+    if (value.length < 32) {
+        return false;
+    }
+    return /^[A-Za-z0-9+/=\s]+$/.test(value);
+}
+function extractLocalMediaHash(url) {
+    const normalized = normalizeLocalUrl(url);
+    if (!normalized) {
+        return null;
+    }
+    const match = normalized.match(/^\/admin\/(media|images)\/([a-f0-9]{16})$/i);
+    return match ? match[2] : null;
+}
+function normalizeLocalUrl(url) {
+    if (url.startsWith("/")) {
+        return url;
+    }
+    try {
+        const parsed = new URL(url);
+        if (!["http:", "https:"].includes(parsed.protocol)) {
+            return null;
+        }
+        if (!["localhost", "127.0.0.1", "::1"].includes(parsed.hostname)) {
+            return null;
+        }
+        return parsed.pathname;
+    }
+    catch {
+        return null;
+    }
+}
+function buildAuth(context, defaultPath) {
+    const authType = context.auth?.type ?? "bearer";
+    if (authType === "none") {
+        return { path: defaultPath, skipDefaultAuth: true };
+    }
+    if (authType === "query") {
+        const keyParam = context.auth?.keyParam ?? "api_key";
+        const apiKey = context.endpoint.apiKey;
+        if (!apiKey) {
+            return { path: defaultPath, skipDefaultAuth: true };
+        }
+        const parsed = new URL(defaultPath, "http://placeholder.local");
+        parsed.searchParams.set(keyParam, apiKey);
+        return { path: `${parsed.pathname}${parsed.search}`, skipDefaultAuth: true };
+    }
+    if (authType === "header") {
+        const headerName = context.auth?.headerName ?? context.auth?.keyParam ?? "x-api-key";
+        const apiKey = context.endpoint.apiKey;
+        if (!apiKey) {
+            return { path: defaultPath, skipDefaultAuth: true };
+        }
+        const prefix = context.auth?.keyPrefix ? `${context.auth.keyPrefix} ` : "";
+        return {
+            path: defaultPath,
+            headers: {
+                [headerName]: `${prefix}${apiKey}`,
+            },
+            skipDefaultAuth: true,
+        };
+    }
+    return { path: defaultPath, skipDefaultAuth: false };
+}
+async function readJsonBody(result) {
+    if (result.rawBody) {
+        return safeJsonParse(result.rawBody.toString("utf8"));
+    }
+    const chunks = [];
+    for await (const chunk of result.body) {
+        chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    }
+    const buffer = Buffer.concat(chunks);
+    return safeJsonParse(buffer.toString("utf8"));
+}
+function safeJsonParse(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch (error) {
+        throw protocolError("invalid_upstream_response", `Inference v2 response is not valid JSON: ${error.message}`, true);
+    }
+}
+function protocolError(type, message, retryable) {
+    const error = new Error(message);
+    error.type = type;
+    error.retryable = retryable;
+    return error;
+}

package/dist/src/protocols/adapters/openai.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.openAiProtocolAdapter = void 0;
+const ALL_OPERATIONS = [
+    "chat_completions",
+    "embeddings",
+    "images_generation",
+    "images_edits",
+    "images_variations",
+    "audio_transcriptions",
+    "audio_translations",
+    "audio_speech",
+];
+exports.openAiProtocolAdapter = {
+    id: "openai",
+    supportedOperations: [...ALL_OPERATIONS],
+    streamSupportedOperations: [...ALL_OPERATIONS],
+    supports(_context) {
+        return { supported: true };
+    },
+    async buildRequest(context) {
+        return {
+            path: context.path,
+            payload: context.payload,
+        };
+    },
+};

package/dist/src/protocols/registry.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.canonicalizeProtocol = canonicalizeProtocol;
+exports.getProtocolAdapter = getProtocolAdapter;
+exports.hasProtocolAdapter = hasProtocolAdapter;
+exports.listAdapterOperations = listAdapterOperations;
+exports.listAllProtocolAdapters = listAllProtocolAdapters;
+exports.routePathToOperation = routePathToOperation;
+const inferenceV2_1 = require("./adapters/inferenceV2");
+const openai_1 = require("./adapters/openai");
+const dashscope_1 = require("./adapters/dashscope");
+const PROTOCOL_ALIASES = {
+    openai: "openai",
+    inference_v2: "inference_v2",
+    "kserve-v2": "inference_v2",
+    kserve_v2: "inference_v2",
+    ray_infer_v2: "inference_v2",
+    "ray-infer-v2": "inference_v2",
+    v2_infer: "inference_v2",
+    "v2-infer": "inference_v2",
+    dashscope: "dashscope",
+};
+const ADAPTERS = new Map([
+    [openai_1.openAiProtocolAdapter.id, openai_1.openAiProtocolAdapter],
+    [inferenceV2_1.inferenceV2ProtocolAdapter.id, inferenceV2_1.inferenceV2ProtocolAdapter],
+    [dashscope_1.dashscopeProtocolAdapter.id, dashscope_1.dashscopeProtocolAdapter],
+]);
+const PROTOCOL_METADATA = {
+    openai: {
+        label: "OpenAI Compatible",
+        description: "Standard OpenAI API format. Supports chat, embeddings, images, and audio.",
+    },
+    inference_v2: {
+        label: "Inference V2 (KServe/Ray)",
+        description: "KServe v2 / Ray Serve inference format. Chat only, no streaming.",
+    },
+    dashscope: {
+        label: "DashScope (Alibaba ModelStudio)",
+        description: "Alibaba Cloud ModelStudio native API. Supports image generation, video generation, and async task-based operations.",
+    },
+};
+function canonicalizeProtocol(raw) {
+    const normalized = (raw ?? "unknown").trim().toLowerCase();
+    return (PROTOCOL_ALIASES[normalized] ?? normalized) || "unknown";
+}
+function getProtocolAdapter(protocol) {
+    const canonical = canonicalizeProtocol(protocol);
+    return ADAPTERS.get(canonical) ?? null;
+}
+function hasProtocolAdapter(protocol) {
+    return getProtocolAdapter(protocol) !== null;
+}
+function listAdapterOperations(protocol) {
+    const adapter = getProtocolAdapter(protocol);
+    if (!adapter) {
+        return null;
+    }
+    return {
+        operations: [...adapter.supportedOperations],
+        streamOperations: [...adapter.streamSupportedOperations],
+    };
+}
+function listAllProtocolAdapters() {
+    return Array.from(ADAPTERS.entries()).map(([id, adapter]) => {
+        const meta = PROTOCOL_METADATA[id] ?? { label: id, description: "" };
+        return {
+            id,
+            label: meta.label,
+            description: meta.description,
+            operations: [...adapter.supportedOperations],
+            streamOperations: [...adapter.streamSupportedOperations],
+            supportsRouting: true,
+        };
+    });
+}
+function routePathToOperation(path) {
+    switch (path) {
+        case "/v1/chat/completions":
+            return "chat_completions";
+        case "/v1/embeddings":
+            return "embeddings";
+        case "/v1/images/generations":
+            return "images_generation";
+        case "/v1/images/edits":
+            return "images_edits";
+        case "/v1/images/variations":
+            return "images_variations";
+        case "/v1/audio/transcriptions":
+            return "audio_transcriptions";
+        case "/v1/audio/translations":
+            return "audio_translations";
+        case "/v1/audio/speech":
+            return "audio_speech";
+        case "/v1/videos/generations":
+            return "video_generations";
+        default:
+            return null;
+    }
+}

package/dist/src/protocols/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/src/providers/health.js

@@ -0,0 +1,153 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultProviderModelHealth = defaultProviderModelHealth;
+exports.getProviderModelHealthMap = getProviderModelHealthMap;
+exports.updateProviderModelHealthCheck = updateProviderModelHealthCheck;
+exports.probeProviderModels = probeProviderModels;
+const undici_1 = require("undici");
+const files_1 = require("../storage/files");
+const repository_1 = require("./repository");
+const repository_2 = require("./repository");
+const DEFAULT_TIMEOUT_MS = 5000;
+function defaultProviderModelHealth() {
+    return {
+        status: "up",
+        consecutiveFailures: 0,
+    };
+}
+async function getProviderModelHealthMap(paths) {
+    const health = await (0, files_1.loadProviderHealth)(paths);
+    return health.models;
+}
+async function updateProviderModelHealthCheck(paths, providerModelId, status, latencyMs, lastStatusCode, lastError) {
+    const health = await (0, files_1.loadProviderHealth)(paths);
+    const now = new Date();
+    const current = health.models[providerModelId] ?? defaultProviderModelHealth();
+    let consecutiveFailures = current.consecutiveFailures;
+    let nextStatus = current.status;
+    if (status === "up") {
+        consecutiveFailures = 0;
+        nextStatus = "up";
+    }
+    else {
+        consecutiveFailures = current.consecutiveFailures + 1;
+        if (consecutiveFailures >= 3) {
+            nextStatus = "down";
+        }
+    }
+    const next = {
+        ...current,
+        status: nextStatus,
+        consecutiveFailures,
+        lastCheckedAt: now,
+        lastStatusCode,
+        lastError,
+    };
+    if (status === "up" && latencyMs !== null) {
+        next.latencyMsEwma = ewma(current.latencyMsEwma, latencyMs);
+        next.lastSuccessAt = now;
+        next.lastFailureAt = undefined;
+        next.lastError = undefined;
+    }
+    else if (status === "down") {
+        next.lastFailureAt = now;
+    }
+    health.models[providerModelId] = next;
+    await (0, files_1.saveProviderHealth)(paths, health);
+}
+async function probeProviderModels(paths, options) {
+    const providers = await (0, repository_1.listProviders)(paths);
+    const targets = collectTargets(providers);
+    const results = [];
+    const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    await Promise.all(Array.from(targets.values()).map(async (target) => {
+        const start = Date.now();
+        try {
+            const dispatcher = target.insecureTls
+                ? new undici_1.Agent({ connect: { rejectUnauthorized: false } })
+                : undefined;
+            const headers = {};
+            if (target.apiKey) {
+                headers.authorization = `Bearer ${target.apiKey}`;
+            }
+            const response = await (0, undici_1.request)(new URL("/v1/models", target.baseUrl).toString(), {
+                method: "GET",
+                headers,
+                headersTimeout: timeoutMs,
+                bodyTimeout: timeoutMs,
+                dispatcher,
+            });
+            const latency = Date.now() - start;
+            response.body.resume();
+            const ok = response.statusCode >= 200 && response.statusCode < 300;
+            for (const model of target.models) {
+                await updateProviderModelHealthCheck(paths, model.providerModelId, ok ? "up" : "down", ok ? latency : null, response.statusCode, ok ? undefined : `status ${response.statusCode}`);
+                results.push({
+                    providerModelId: model.providerModelId,
+                    providerId: model.providerId,
+                    modelId: model.modelId,
+                    baseUrl: target.baseUrl,
+                    status: ok ? "up" : "down",
+                    latencyMs: ok ? latency : null,
+                    statusCode: response.statusCode,
+                    error: ok ? undefined : `status ${response.statusCode}`,
+                });
+            }
+        }
+        catch (error) {
+            const errorMsg = error.message || "unknown error";
+            for (const model of target.models) {
+                await updateProviderModelHealthCheck(paths, model.providerModelId, "down", null, undefined, errorMsg);
+                results.push({
+                    providerModelId: model.providerModelId,
+                    providerId: model.providerId,
+                    modelId: model.modelId,
+                    baseUrl: target.baseUrl,
+                    status: "down",
+                    latencyMs: null,
+                    error: errorMsg,
+                });
+            }
+        }
+    }));
+    return results;
+}
+function collectTargets(providers) {
+    const targets = new Map();
+    for (const provider of providers) {
+        if (!provider.enabled) {
+            continue;
+        }
+        for (const model of provider.models) {
+            if (model.enabled === false) {
+                continue;
+            }
+            const baseUrl = model.baseUrl ?? provider.baseUrl;
+            if (!baseUrl) {
+                continue;
+            }
+            const apiKey = model.apiKey ?? provider.apiKey;
+            const insecureTls = (0, repository_2.getEffectiveModelInsecureTls)(provider, model);
+            const key = `${baseUrl}@@${apiKey ? "key" : "nokey"}@@${insecureTls ? "insecure" : "secure"}`;
+            const entry = targets.get(key) ?? {
+                baseUrl,
+                apiKey,
+                insecureTls,
+                models: [],
+            };
+            entry.models.push({
+                providerModelId: model.providerModelId,
+                providerId: provider.id,
+                modelId: model.modelId,
+            });
+            targets.set(key, entry);
+        }
+    }
+    return targets;
+}
+function ewma(previous, next, alpha = 0.2) {
+    if (previous === undefined) {
+        return next;
+    }
+    return alpha * next + (1 - alpha) * previous;
+}