waypoi 0.0.0

package/src/utils/messageMedia.ts

@@ -0,0 +1,293 @@
+import path from "path";
+import { StoragePaths } from "../storage/files";
+import { getMediaEntry, getMediaPath } from "../storage/imageCache";
+
+export interface MessageMediaScan {
+  hasImage: boolean;
+  hasAudio: boolean;
+}
+
+export function scanMessageModalities(messages: unknown): MessageMediaScan {
+  const result: MessageMediaScan = { hasImage: false, hasAudio: false };
+  if (!Array.isArray(messages)) {
+    return result;
+  }
+
+  for (const message of messages) {
+    if (!message || typeof message !== "object") {
+      continue;
+    }
+    const content = (message as { content?: unknown }).content;
+    if (!Array.isArray(content)) {
+      continue;
+    }
+    for (const part of content) {
+      if (!part || typeof part !== "object") {
+        continue;
+      }
+      const type = (part as { type?: unknown }).type;
+      if (type === "image_url" || type === "input_image" || type === "image") {
+        result.hasImage = true;
+      }
+      if (type === "input_audio" || type === "audio") {
+        result.hasAudio = true;
+      }
+    }
+  }
+
+  return result;
+}
+
+export async function normalizeMessagesForUpstream(
+  paths: StoragePaths,
+  messages: unknown
+): Promise<unknown> {
+  if (!Array.isArray(messages)) {
+    return messages;
+  }
+
+  const normalized: unknown[] = [];
+  for (const message of messages) {
+    if (!message || typeof message !== "object") {
+      normalized.push(message);
+      continue;
+    }
+
+    const nextMessage: Record<string, unknown> = { ...(message as Record<string, unknown>) };
+    const content = nextMessage.content;
+    if (!Array.isArray(content)) {
+      normalized.push(nextMessage);
+      continue;
+    }
+
+    const nextContent: unknown[] = [];
+    for (const rawPart of content) {
+      if (!rawPart || typeof rawPart !== "object") {
+        nextContent.push(rawPart);
+        continue;
+      }
+      const part = { ...(rawPart as Record<string, unknown>) };
+      const type = part.type;
+
+      if (type === "video") {
+        throw invalidRequestError("Video content is not supported in v1 omni mode.");
+      }
+
+      if (type === "image" && typeof part.image === "string") {
+        nextContent.push({ type: "image_url", image_url: { url: part.image } });
+        continue;
+      }
+
+      if (type === "image_url") {
+        nextContent.push(await normalizeImageUrlPart(paths, part));
+        continue;
+      }
+
+      if (type === "audio" && typeof part.audio === "string") {
+        nextContent.push(await normalizeAudioValue(paths, part.audio));
+        continue;
+      }
+
+      if (type === "input_audio") {
+        nextContent.push(await normalizeInputAudioPart(paths, part));
+        continue;
+      }
+
+      nextContent.push(part);
+    }
+
+    nextMessage.content = nextContent;
+    normalized.push(nextMessage);
+  }
+
+  return normalized;
+}
+
+async function normalizeImageUrlPart(
+  paths: StoragePaths,
+  part: Record<string, unknown>
+): Promise<Record<string, unknown>> {
+  const imageUrlObject = (part.image_url ?? {}) as { url?: unknown };
+  const value = imageUrlObject.url;
+  if (typeof value !== "string" || value.length === 0) {
+    return part;
+  }
+
+  if (value.startsWith("data:")) {
+    return part;
+  }
+
+  const hash = extractLocalHash(value);
+  if (!hash) {
+    return part;
+  }
+
+  const mediaPath = await getMediaPath(paths, hash);
+  const mediaEntry = await getMediaEntry(paths, hash);
+  if (!mediaPath || !mediaEntry) {
+    throw invalidRequestError("Referenced image not found in cache.");
+  }
+  const file = await import("fs/promises");
+  const buffer = await file.readFile(mediaPath);
+  const dataUrl = `data:${mediaEntry.mimeType};base64,${buffer.toString("base64")}`;
+
+  return {
+    ...part,
+    type: "image_url",
+    image_url: {
+      ...imageUrlObject,
+      url: dataUrl,
+    },
+  };
+}
+
+async function normalizeInputAudioPart(
+  paths: StoragePaths,
+  part: Record<string, unknown>
+): Promise<Record<string, unknown>> {
+  const inputAudio = (part.input_audio ?? {}) as Record<string, unknown>;
+  const data = inputAudio.data;
+  const url = inputAudio.url;
+
+  if (typeof data === "string" && data.length > 0) {
+    return {
+      ...part,
+      type: "input_audio",
+      input_audio: {
+        ...inputAudio,
+      },
+    };
+  }
+
+  if (typeof url === "string" && url.length > 0) {
+    const resolved = await resolveLocalMediaUrl(paths, url);
+    return {
+      ...part,
+      type: "input_audio",
+      input_audio: resolved,
+    };
+  }
+
+  throw invalidRequestError("input_audio requires either data or a local media url.");
+}
+
+async function normalizeAudioValue(paths: StoragePaths, value: string): Promise<Record<string, unknown>> {
+  if (value.startsWith("data:")) {
+    const parsed = parseDataUrl(value);
+    return {
+      type: "input_audio",
+      input_audio: {
+        data: parsed.base64,
+        format: parsed.format,
+      },
+    };
+  }
+
+  if (looksLikeBase64(value)) {
+    return {
+      type: "input_audio",
+      input_audio: {
+        data: value,
+      },
+    };
+  }
+
+  const resolved = await resolveLocalMediaUrl(paths, value);
+  return {
+    type: "input_audio",
+    input_audio: resolved,
+  };
+}
+
+async function resolveLocalMediaUrl(
+  paths: StoragePaths,
+  url: string
+): Promise<{ data: string; format?: string }> {
+  const hash = extractLocalHash(url);
+  if (!hash) {
+    throw invalidRequestError("Only local /admin/media or /admin/images URLs are allowed for input_audio.");
+  }
+
+  const mediaPath = await getMediaPath(paths, hash);
+  const mediaEntry = await getMediaEntry(paths, hash);
+  if (!mediaPath || !mediaEntry) {
+    throw invalidRequestError("Referenced media not found in cache.");
+  }
+
+  const file = await import("fs/promises");
+  const buffer = await file.readFile(mediaPath);
+  return {
+    data: buffer.toString("base64"),
+    format: audioFormatFromMime(mediaEntry.mimeType, mediaPath),
+  };
+}
+
+function extractLocalHash(url: string): string | null {
+  const normalized = normalizeLocalUrl(url);
+  if (!normalized) {
+    return null;
+  }
+  const mediaMatch = normalized.match(/^\/admin\/(media|images)\/([a-f0-9]{16})$/i);
+  if (!mediaMatch) {
+    return null;
+  }
+  return mediaMatch[2];
+}
+
+function normalizeLocalUrl(url: string): string | null {
+  if (url.startsWith("/")) {
+    return url;
+  }
+
+  try {
+    const parsed = new URL(url);
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+      return null;
+    }
+    if (!["localhost", "127.0.0.1", "::1"].includes(parsed.hostname)) {
+      return null;
+    }
+    return parsed.pathname;
+  } catch {
+    return null;
+  }
+}
+
+function parseDataUrl(value: string): { base64: string; format?: string } {
+  const match = value.match(/^data:([^;]+);base64,(.+)$/i);
+  if (!match) {
+    throw invalidRequestError("Invalid data URL for audio input.");
+  }
+  return {
+    base64: match[2].replace(/\s+/g, ""),
+    format: audioFormatFromMime(match[1]),
+  };
+}
+
+function audioFormatFromMime(mimeType: string, filePath?: string): string | undefined {
+  const lower = mimeType.toLowerCase();
+  if (lower.includes("wav")) return "wav";
+  if (lower.includes("mpeg") || lower.includes("mp3")) return "mp3";
+  if (lower.includes("ogg")) return "ogg";
+  if (lower.includes("webm")) return "webm";
+  if (lower.includes("mp4") || lower.includes("m4a")) return "m4a";
+  if (filePath) {
+    const ext = path.extname(filePath).slice(1).toLowerCase();
+    if (ext) return ext;
+  }
+  return undefined;
+}
+
+function looksLikeBase64(value: string): boolean {
+  if (value.length < 32) {
+    return false;
+  }
+  return /^[A-Za-z0-9+/=\s]+$/.test(value);
+}
+
+function invalidRequestError(message: string): Error & { type: string; retryable: boolean } {
+  const error = new Error(message) as Error & { type: string; retryable: boolean };
+  error.type = "invalid_request";
+  error.retryable = false;
+  return error;
+}

package/src/utils/modelCapabilities.ts

@@ -0,0 +1,161 @@
+import {
+  EndpointType,
+  ModelCapabilities,
+  ModelMapping,
+  ModelModality,
+} from "../types";
+
+export interface CapabilitiesRequirements {
+  requiredInput?: ModelModality[];
+  requiredOutput?: ModelModality[];
+}
+
+export function resolveCapabilities(
+  mapping: ModelMapping,
+  endpointType: EndpointType,
+  upstreamCaps?: ModelCapabilities
+): ModelCapabilities {
+  if (mapping.capabilities) {
+    return normalizeCapabilities(mapping.capabilities, "configured");
+  }
+  if (upstreamCaps) {
+    return normalizeCapabilities(upstreamCaps, "inferred");
+  }
+
+  const inferred = inferCapabilities(mapping.publicName, endpointType);
+  warnInference(mapping.publicName, endpointType, inferred);
+  return normalizeCapabilities(inferred, "inferred");
+}
+
+export function inferCapabilities(
+  modelName: string,
+  endpointType: EndpointType
+): ModelCapabilities {
+  const name = modelName.toLowerCase();
+
+  if (endpointType === "embedding") {
+    return { input: ["text"], output: ["embedding"] };
+  }
+
+  if (endpointType === "diffusion") {
+    return { input: ["text"], output: ["image"] };
+  }
+
+  if (endpointType === "audio") {
+    if (isTtsModelName(name)) {
+      return { input: ["text"], output: ["audio"] };
+    }
+    return { input: ["audio"], output: ["text"] };
+  }
+
+  if (endpointType === "video") {
+    if (isImageToVideoModelName(name)) {
+      return { input: ["text", "image"], output: ["video"] };
+    }
+    return { input: ["text"], output: ["video"] };
+  }
+
+  if (isVisionModelName(name)) {
+    return { input: ["text", "image"], output: ["text"], supportsTools: true, supportsStreaming: true };
+  }
+
+  return { input: ["text"], output: ["text"], supportsTools: true, supportsStreaming: true };
+}
+
+export function supportsRequirements(
+  capabilities: ModelCapabilities,
+  requirements?: CapabilitiesRequirements
+): boolean {
+  if (!requirements) {
+    return true;
+  }
+
+  if (requirements.requiredInput && requirements.requiredInput.length > 0) {
+    for (const modality of requirements.requiredInput) {
+      if (!capabilities.input.includes(modality)) {
+        return false;
+      }
+    }
+  }
+
+  if (requirements.requiredOutput && requirements.requiredOutput.length > 0) {
+    for (const modality of requirements.requiredOutput) {
+      if (!capabilities.output.includes(modality)) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+function normalizeCapabilities(
+  capabilities: ModelCapabilities,
+  source: "configured" | "inferred"
+): ModelCapabilities {
+  return {
+    input: normalizeModalities(capabilities.input),
+    output: normalizeModalities(capabilities.output),
+    supportsTools: capabilities.supportsTools,
+    supportsStreaming: capabilities.supportsStreaming,
+    source,
+  };
+}
+
+function normalizeModalities(modalities: ModelModality[]): ModelModality[] {
+  const allowed: ModelModality[] = ["text", "image", "audio", "embedding", "video"];
+  const unique = new Set<ModelModality>();
+
+  for (const modality of modalities) {
+    if (allowed.includes(modality)) {
+      unique.add(modality);
+    }
+  }
+
+  return allowed.filter((modality) => unique.has(modality));
+}
+
+function isTtsModelName(name: string): boolean {
+  return (
+    name.includes("tts") ||
+    name.includes("speech") ||
+    name.includes("voice") ||
+    name.includes("audio-gen")
+  );
+}
+
+function isVisionModelName(name: string): boolean {
+  return (
+    name.includes("vision") ||
+    name.includes("vl") ||
+    name.includes("omni") ||
+    name.includes("multimodal")
+  );
+}
+
+function isImageToVideoModelName(name: string): boolean {
+  return (
+    name.includes("i2v") ||
+    name.includes("image-to-video") ||
+    name.includes("img2vid") ||
+    name.includes("kf2v")
+  );
+}
+
+const capabilityInferenceWarnings = new Set<string>();
+
+function warnInference(
+  modelName: string,
+  endpointType: EndpointType,
+  capabilities: ModelCapabilities
+): void {
+  const key = `${endpointType}:${modelName}`;
+  if (capabilityInferenceWarnings.has(key)) {
+    return;
+  }
+  capabilityInferenceWarnings.add(key);
+  console.warn(
+    `[waypoi] Inferred capabilities for model '${modelName}' on ${endpointType}: ` +
+      `${capabilities.input.join("+")}->${capabilities.output.join("+")}`
+  );
+}

package/src/utils/modelDiscovery.ts

@@ -0,0 +1,203 @@
+import { Agent, request } from "undici";
+import { ModelCapabilities, ModelMapping, ModelModality } from "../types";
+import { ProviderAuthConfig } from "../providers/types";
+
+export interface EndpointInfo {
+  baseUrl: string;
+  apiKey?: string;
+  insecureTls: boolean;
+  auth?: ProviderAuthConfig;
+}
+
+export async function resolveModelMappings(
+  endpoint: EndpointInfo,
+  mappings: ModelMapping[]
+): Promise<ModelMapping[]> {
+  let models: UpstreamModelInfo[] | null = null;
+  try {
+    models = await discoverUpstreamModels(endpoint);
+  } catch {
+    models = null;
+  }
+  if (!models || models.length === 0) {
+    return mappings;
+  }
+  
+  // If exactly one model, use it as the upstream for all mappings
+  // that don't already have an explicit upstream different from the public name
+  if (models.length === 1) {
+    const sole = models[0];
+    console.log(`[model-discovery] Single model found: ${sole.id}`);
+    return mappings.map((mapping) => {
+      // If user specified explicit upstream (public=upstream format), keep it
+      // Otherwise, use the discovered model as upstream
+      if (mapping.publicName !== mapping.upstreamModel) {
+        // User specified explicit mapping, keep it
+        return mapping;
+      }
+      // Public and upstream are the same (user just gave public name)
+      // Replace upstream with discovered model
+      console.log(`[model-discovery] Mapping ${mapping.publicName} -> ${sole.id}`);
+      return {
+        ...mapping,
+        upstreamModel: sole.id,
+        capabilities: mapping.capabilities ?? sole.capabilities,
+      };
+    });
+  }
+  
+  // Multiple models - check if any mapping's upstream matches available models
+  console.log(
+    `[model-discovery] ${models.length} models found: ${models
+      .slice(0, 5)
+      .map((model) => model.id)
+      .join(", ")}${models.length > 5 ? "..." : ""}`
+  );
+  const byId = new Map(models.map((model) => [model.id, model]));
+  return mappings.map((mapping) => {
+    if (mapping.capabilities) {
+      return mapping;
+    }
+    const matched = byId.get(mapping.upstreamModel) ?? byId.get(mapping.publicName);
+    if (!matched?.capabilities) {
+      return mapping;
+    }
+    return {
+      ...mapping,
+      capabilities: matched.capabilities,
+    };
+  });
+}
+
+interface UpstreamModelInfo {
+  id: string;
+  capabilities?: ModelCapabilities;
+}
+
+export async function discoverUpstreamModels(endpoint: EndpointInfo): Promise<UpstreamModelInfo[]> {
+  const dispatcher = endpoint.insecureTls
+    ? new Agent({ connect: { rejectUnauthorized: false } })
+    : undefined;
+  const { url, headers } = buildDiscoveryRequest(endpoint);
+  const response = await request(url, {
+    method: "GET",
+    headersTimeout: 3000,
+    bodyTimeout: 3000,
+    dispatcher,
+    headers,
+  });
+  const body = (await readJson(response.body)) as {
+    data?: Array<{
+      id?: string;
+      input_modalities?: string[];
+      output_modalities?: string[];
+      capabilities?: { input?: string[]; output?: string[]; supportsTools?: boolean; supportsStreaming?: boolean };
+    }>;
+  } | null;
+  response.body.resume();
+  if (response.statusCode < 200 || response.statusCode >= 300) {
+    throw new Error(`model discovery failed with status ${response.statusCode}`);
+  }
+  const list = Array.isArray(body?.data) ? body.data : [];
+  const models: UpstreamModelInfo[] = [];
+  for (const item of list) {
+    if (!item.id) {
+      continue;
+    }
+    const modelInfo: UpstreamModelInfo = { id: item.id };
+    const capabilities = extractCapabilities(item);
+    if (capabilities) {
+      modelInfo.capabilities = capabilities;
+    }
+    models.push(modelInfo);
+  }
+  return models;
+}
+
+function buildDiscoveryRequest(endpoint: EndpointInfo): { url: string; headers: Record<string, string> } {
+  const authType = endpoint.auth?.type ?? "bearer";
+  const headers: Record<string, string> = {};
+  const url = new URL(buildModelListUrl(endpoint.baseUrl));
+  const apiKey = endpoint.apiKey?.trim();
+
+  if (apiKey && authType === "query") {
+    const keyParam = endpoint.auth?.keyParam?.trim() || "api_key";
+    url.searchParams.set(keyParam, apiKey);
+  } else if (apiKey && authType === "header") {
+    const headerName = endpoint.auth?.headerName?.trim() || endpoint.auth?.keyParam?.trim() || "x-api-key";
+    const prefix = endpoint.auth?.keyPrefix?.trim();
+    headers[headerName] = prefix ? `${prefix} ${apiKey}` : apiKey;
+  } else if (apiKey && authType !== "none") {
+    headers.authorization = `Bearer ${apiKey}`;
+  }
+
+  return { url: url.toString(), headers };
+}
+
+function buildModelListUrl(baseUrl: string): string {
+  const parsed = new URL(baseUrl);
+  const pathname = parsed.pathname.replace(/\/+$/, "");
+  if (!pathname) {
+    parsed.pathname = "/v1/models";
+  } else if (pathname.endsWith("/v1")) {
+    parsed.pathname = `${pathname}/models`;
+  } else {
+    parsed.pathname = `${pathname}/v1/models`;
+  }
+  return parsed.toString();
+}
+
+function extractCapabilities(item: {
+  input_modalities?: string[];
+  output_modalities?: string[];
+  capabilities?: { input?: string[]; output?: string[]; supportsTools?: boolean; supportsStreaming?: boolean };
+}): ModelCapabilities | undefined {
+  const fromCapabilities = item.capabilities;
+  if (fromCapabilities?.input && fromCapabilities?.output) {
+    const input = normalizeModalities(fromCapabilities.input);
+    const output = normalizeModalities(fromCapabilities.output);
+    if (input.length > 0 && output.length > 0) {
+      return {
+        input,
+        output,
+        supportsTools: fromCapabilities.supportsTools,
+        supportsStreaming: fromCapabilities.supportsStreaming,
+        source: "inferred",
+      };
+    }
+  }
+
+  const input = normalizeModalities(item.input_modalities ?? []);
+  const output = normalizeModalities(item.output_modalities ?? []);
+  if (input.length > 0 && output.length > 0) {
+    return { input, output, source: "inferred" };
+  }
+
+  return undefined;
+}
+
+function normalizeModalities(values: string[]): ModelModality[] {
+  const normalized = new Set<ModelModality>();
+  for (const value of values) {
+    const lower = value.toLowerCase();
+    if (lower === "text" || lower === "image" || lower === "audio" || lower === "embedding") {
+      normalized.add(lower);
+    }
+  }
+  return Array.from(normalized);
+}
+
+async function readJson(stream: NodeJS.ReadableStream): Promise<unknown> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  if (chunks.length === 0) {
+    return null;
+  }
+  try {
+    return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+  } catch {
+    return null;
+  }
+}

package/src/workers/captureRetention.ts

@@ -0,0 +1,25 @@
+import { StoragePaths } from "../storage/files";
+import { runCaptureRetention } from "../storage/captureRepository";
+
+let retentionTimer: NodeJS.Timeout | null = null;
+
+export function startCaptureRetentionWorker(paths: StoragePaths): void {
+  const run = async () => {
+    try {
+      await runCaptureRetention(paths);
+    } catch {
+      // ignore background errors
+    }
+  };
+
+  retentionTimer = setInterval(run, 10 * 60 * 1000);
+  retentionTimer.unref();
+  void run();
+}
+
+export function stopCaptureRetentionWorker(): void {
+  if (retentionTimer) {
+    clearInterval(retentionTimer);
+    retentionTimer = null;
+  }
+}