waypoi 0.0.0

package/src/mcp/service.ts

@@ -0,0 +1,611 @@
+import { randomUUID } from "crypto";
+import { IncomingMessage, ServerResponse } from "http";
+import { promises as fs } from "fs";
+import path from "path";
+import { z } from "zod";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
+import { StoragePaths } from "../storage/files";
+import {
+  normalizeImageGenerationPayload,
+  runImageGeneration,
+} from "../services/imageGeneration";
+import { imageDataUrlFromPath, runImageUnderstanding } from "../services/imageUnderstanding";
+import { resolveVideoGenerationModel, runVideoGeneration } from "../services/videoGeneration";
+import { ImageGenerationRequest, VideoGenerationRequest } from "../types";
+import {
+  validateAtMostOneImageInput,
+  resolveBinaryOutputPolicy,
+  typedError,
+  validateSingleImageInput,
+} from "./policy";
+
+interface McpSessionEntry {
+  server: McpServer;
+  transport: StreamableHTTPServerTransport;
+}
+
+export interface McpServiceDependencies {
+  runImageGeneration: typeof runImageGeneration;
+  normalizeImageGenerationPayload: typeof normalizeImageGenerationPayload;
+  runImageUnderstanding: typeof runImageUnderstanding;
+  runVideoGeneration: typeof runVideoGeneration;
+  resolveVideoGenerationModel: typeof resolveVideoGenerationModel;
+}
+
+export type McpServiceDependencyOverrides = Partial<McpServiceDependencies>;
+
+export interface McpService {
+  handleRequest(req: IncomingMessage, res: ServerResponse, parsedBody?: unknown): Promise<void>;
+  close(): Promise<void>;
+}
+
+const defaultDeps: McpServiceDependencies = {
+  runImageGeneration,
+  normalizeImageGenerationPayload,
+  runImageUnderstanding,
+  runVideoGeneration,
+  resolveVideoGenerationModel,
+};
+
+const GENERATE_IMAGE_TOOL_DESCRIPTION =
+  "Generate or edit images from a text prompt. Supports text-to-image and image-to-image editing. Successful calls always write files to the waypoi config directory (~/.config/waypoi/generated-images by default; override with WAYPOI_MCP_OUTPUT_ROOT). Report file_path or file_paths to the user. Use include_data=true only when inline image data is also needed.";
+
+const UNDERSTAND_IMAGE_TOOL_DESCRIPTION =
+  "Analyze one image and return structured text. Provide exactly one of image_path (local file) or image_url (http/https or data URL). Use the instruction parameter to specify what analysis you need (e.g., 'describe the image', 'find all objects and their bounding boxes', 'extract text via OCR'). If returning points or boxes, use original-image pixel coordinates. Keep instruction concise and specify the output format you expect.";
+
+const GENERATE_VIDEO_TOOL_DESCRIPTION =
+  "Generate videos from text prompts or images. Supports text-to-video and image-to-video generation using Alibaba Cloud ModelStudio (Wan models). Videos are generated asynchronously and may take 1-5 minutes. Returns a URL to the generated MP4 video (H.264 encoding). Provide a detailed prompt describing the desired video content, style, and camera movement. Optionally provide an image_url to use as the first frame for image-to-video generation.";
+
+export function createMcpService(
+  paths: StoragePaths,
+  deps: McpServiceDependencyOverrides = {}
+): McpService {
+  const resolvedDeps: McpServiceDependencies = {
+    ...defaultDeps,
+    ...deps,
+  };
+  const sessions = new Map<string, McpSessionEntry>();
+
+  const createServer = (): McpServer => {
+    const server = new McpServer(
+      {
+        name: "waypoi-mcp",
+        version: "0.7.1",
+      },
+      {
+        capabilities: {},
+      }
+    );
+
+    server.registerTool(
+      "generate_image",
+      {
+        description: GENERATE_IMAGE_TOOL_DESCRIPTION,
+        inputSchema: {
+          prompt: z.string().min(1),
+          model: z.string().optional(),
+          image_path: z.string().optional(),
+          image_url: z.string().optional(),
+          n: z.number().int().min(1).max(4).optional(),
+          size: z.string().optional(),
+          quality: z.string().optional(),
+          style: z.string().optional(),
+          response_format: z.enum(["url", "b64_json"]).optional(),
+          include_data: z.boolean().optional(),
+        },
+      },
+      async (args) => {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 60_000);
+        try {
+          validateAtMostOneImageInput({
+            image_path: args.image_path,
+            image_url: args.image_url,
+          });
+          const resolvedImageUrl = await resolveOptionalImageInputToUrl({
+            image_path: args.image_path,
+            image_url: args.image_url,
+          });
+          const filePolicy = resolveBinaryOutputPolicy(
+            { include_data: args.include_data },
+            { defaultBaseDir: paths.baseDir }
+          );
+          const hasFileOutput = true;
+          // File output requires decodable bytes; force b64_json upstream even if caller asks for "url".
+          const responseFormat = hasFileOutput ? "b64_json" : (args.response_format ?? "b64_json");
+          const request: ImageGenerationRequest = {
+            prompt: args.prompt,
+            model: args.model,
+            n: args.n,
+            size: args.size,
+            quality: args.quality,
+            style: args.style,
+            response_format: responseFormat,
+            image_url: resolvedImageUrl,
+          };
+
+          const generated = await resolvedDeps.runImageGeneration(paths, request, {}, controller.signal);
+          const normalized = await resolvedDeps.normalizeImageGenerationPayload(
+            paths,
+            generated.payload,
+            generated.model
+          );
+          const artifacts = await materializeImagesToFiles(normalized.images, normalized.created, {
+            outputDir: filePolicy.outputDir,
+            includeData: filePolicy.includeData,
+            outputBaseRoot: filePolicy.outputBaseRoot,
+          });
+          const filePaths = artifacts.map((artifact) => artifact.file_path);
+          const summary = buildGenerateImageSummary(filePaths.length);
+
+          const output = {
+            ok: true,
+            summary,
+            model: normalized.model,
+            created: normalized.created,
+            ...(filePaths.length === 1
+              ? { file_path: filePaths[0] }
+              : { file_paths: filePaths }),
+            artifacts,
+          };
+          return {
+            content: [
+              {
+                type: "text" as const,
+                text: JSON.stringify(buildGenerateImageTextPayload(output)),
+              },
+            ],
+            structuredContent: output,
+          };
+        } catch (error) {
+          const typed = error as Error & { type?: string };
+          const type = typed.type ?? "upstream_error";
+          const message =
+            type === "no_diffusion_model"
+              ? "No diffusion model available. Add or enable a provider model."
+              : typed.message || "Image generation failed";
+          const output = {
+            ok: false,
+            error: {
+              type,
+              message,
+            },
+          };
+          return {
+            isError: true,
+            content: [
+              {
+                type: "text" as const,
+                text: JSON.stringify(output),
+              },
+            ],
+            structuredContent: output,
+          };
+        } finally {
+          clearTimeout(timeout);
+        }
+      }
+    );
+
+    server.registerTool(
+      "understand_image",
+      {
+        description: UNDERSTAND_IMAGE_TOOL_DESCRIPTION,
+        inputSchema: {
+          image_path: z.string().optional(),
+          image_url: z.string().optional(),
+          instruction: z.string().optional(),
+          model: z.string().optional(),
+          max_tokens: z.number().int().min(1).max(4096).optional(),
+          temperature: z.number().min(0).max(2).optional(),
+        },
+      },
+      async (args) => {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 60_000);
+        try {
+          validateSingleImageInput({
+            image_path: args.image_path,
+            image_url: args.image_url,
+          });
+          const result = await resolvedDeps.runImageUnderstanding(
+            paths,
+            {
+              image_path: args.image_path,
+              image_url: args.image_url,
+              instruction: args.instruction,
+              model: args.model,
+              max_tokens: args.max_tokens,
+              temperature: args.temperature,
+            },
+            controller.signal
+          );
+          const summary = "Image analyzed.";
+          const output = {
+            ok: true,
+            summary,
+            model: result.model,
+            text: result.raw_text,
+            result: result.analysis,
+            ...(result.image_geometry ? { image_geometry: result.image_geometry } : {}),
+            ...(result.usage ? { usage: result.usage } : {}),
+          };
+          return {
+            content: [
+              {
+                type: "text" as const,
+                text: JSON.stringify({
+                  ok: true,
+                  summary,
+                  model: result.model,
+                  text: result.raw_text,
+                }),
+              },
+            ],
+            structuredContent: output,
+          };
+        } catch (error) {
+          const typed = error as Error & { type?: string };
+          const type = typed.type ?? "upstream_error";
+          const output = {
+            ok: false,
+            error: {
+              type,
+              message: typed.message || "Image understanding failed",
+            },
+          };
+          return {
+            isError: true,
+            content: [{ type: "text" as const, text: JSON.stringify(output) }],
+            structuredContent: output,
+          };
+        } finally {
+          clearTimeout(timeout);
+        }
+      }
+    );
+
+    server.registerTool(
+      "generate_video",
+      {
+        description: GENERATE_VIDEO_TOOL_DESCRIPTION,
+        inputSchema: {
+          prompt: z.string().min(1),
+          model: z.string().optional(),
+          image_url: z.string().optional(),
+          audio_url: z.string().optional(),
+          duration: z.number().int().min(2).max(15).optional(),
+          resolution: z.enum(["480P", "720P", "1080P"]).optional(),
+          negative_prompt: z.string().optional(),
+          seed: z.number().int().min(0).max(2147483647).optional(),
+          watermark: z.boolean().optional(),
+          prompt_extend: z.boolean().optional(),
+        },
+      },
+      async (args) => {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 300_000);
+        try {
+          const resolvedModel = await resolvedDeps.resolveVideoGenerationModel(paths, args.model);
+          if (!resolvedModel) {
+            throw typedError("no_video_model", "No video generation model available. Add or enable a provider model.");
+          }
+          const request: VideoGenerationRequest = {
+            prompt: args.prompt,
+            model: resolvedModel,
+            image_url: args.image_url,
+            audio_url: args.audio_url,
+            duration: args.duration,
+            resolution: args.resolution,
+            negative_prompt: args.negative_prompt,
+            seed: args.seed,
+            watermark: args.watermark,
+            prompt_extend: args.prompt_extend,
+          };
+
+          const generated = await resolvedDeps.runVideoGeneration(paths, request, {}, controller.signal);
+          const payload = generated.payload as { data?: Array<{ url?: string; revised_prompt?: string }>; usage?: { video_count?: number; duration?: number; resolution?: string } };
+          const data = payload.data ?? [];
+          const usage = payload.usage ?? {};
+
+          if (data.length === 0) {
+            throw typedError("no_video_output", "Video generation completed but no video URL was returned.");
+          }
+
+          const videoData = data[0];
+          const output = {
+            ok: true,
+            summary: "Generated 1 video.",
+            model: generated.route.upstreamModel,
+            url: videoData.url,
+            ...(videoData.revised_prompt ? { revised_prompt: videoData.revised_prompt } : {}),
+            ...(usage.video_count ? { video_count: usage.video_count } : {}),
+            ...(usage.duration ? { duration: usage.duration } : {}),
+            ...(usage.resolution ? { resolution: usage.resolution } : {}),
+          };
+          return {
+            content: [
+              {
+                type: "text" as const,
+                text: JSON.stringify(output),
+              },
+            ],
+            structuredContent: output,
+          };
+        } catch (error) {
+          const typed = error as Error & { type?: string };
+          const type = typed.type ?? "upstream_error";
+          const message =
+            type === "no_video_model"
+              ? "No video generation model available. Add or enable a provider model."
+              : typed.message || "Video generation failed";
+          const output = {
+            ok: false,
+            error: {
+              type,
+              message,
+            },
+          };
+          return {
+            isError: true,
+            content: [
+              {
+                type: "text" as const,
+                text: JSON.stringify(output),
+              },
+            ],
+            structuredContent: output,
+          };
+        } finally {
+          clearTimeout(timeout);
+        }
+      }
+    );
+
+    return server;
+  };
+
+  const close = async (): Promise<void> => {
+    const entries = Array.from(sessions.values());
+    sessions.clear();
+    await Promise.allSettled(
+      entries.map(async (entry) => {
+        await entry.transport.close();
+        await entry.server.close();
+      })
+    );
+  };
+
+  const handleRequest = async (
+    req: IncomingMessage,
+    res: ServerResponse,
+    parsedBody?: unknown
+  ): Promise<void> => {
+    const sessionIdHeader = req.headers["mcp-session-id"];
+    const sessionId =
+      typeof sessionIdHeader === "string"
+        ? sessionIdHeader
+        : Array.isArray(sessionIdHeader)
+          ? sessionIdHeader[0]
+          : undefined;
+
+    let entry: McpSessionEntry | undefined;
+    if (sessionId) {
+      entry = sessions.get(sessionId);
+    }
+
+    if (!entry) {
+      if (sessionId || !isInitializeRequest(parsedBody)) {
+        if (!res.headersSent) {
+          res.writeHead(400, { "Content-Type": "application/json" });
+          res.end(
+            JSON.stringify({
+              jsonrpc: "2.0",
+              error: {
+                code: -32000,
+                message: "Bad Request: No valid MCP session ID provided",
+              },
+              id: null,
+            })
+          );
+        }
+        return;
+      }
+
+      const server = createServer();
+      const transport = new StreamableHTTPServerTransport({
+        sessionIdGenerator: () => randomUUID(),
+        enableJsonResponse: true,
+        onsessioninitialized: (newSessionId) => {
+          sessions.set(newSessionId, { server, transport });
+        },
+        onsessionclosed: (closedSessionId) => {
+          sessions.delete(closedSessionId);
+        },
+      });
+      transport.onclose = () => {
+        const sid = transport.sessionId;
+        if (sid) {
+          sessions.delete(sid);
+        }
+      };
+      await server.connect(transport);
+      entry = { server, transport };
+    }
+
+    await entry.transport.handleRequest(req, res, parsedBody);
+  };
+
+  return {
+    handleRequest,
+    close,
+  };
+}
+
+type FileMaterializeOptions = {
+  outputDir: string;
+  includeData: boolean;
+  outputBaseRoot: string;
+};
+
+async function materializeImagesToFiles(
+  images: Array<{ index: number; url?: string; b64_json?: string; revised_prompt?: string }>,
+  created: number,
+  options: FileMaterializeOptions
+): Promise<
+  Array<{
+    index: number;
+    file_path: string;
+    mime_type: string;
+    bytes: number;
+    revised_prompt?: string;
+    url?: string;
+    b64_json?: string;
+  }>
+> {
+  const output: Array<{
+    index: number;
+    file_path: string;
+    mime_type: string;
+    bytes: number;
+    revised_prompt?: string;
+    url?: string;
+    b64_json?: string;
+  }> = [];
+
+  for (const image of images) {
+    const payload = decodeImagePayload(image);
+    if (!payload) {
+      throw typedError(
+        "invalid_request",
+        `Image ${image.index} has no decodable bytes for file output.`
+      );
+    }
+    const extension = extensionForMime(payload.mimeType);
+    const resolvedPath = resolveOutputPath(image.index, created, extension, options);
+    await fs.mkdir(path.dirname(resolvedPath), { recursive: true });
+    await fs.writeFile(resolvedPath, payload.buffer);
+
+    const entry: {
+      index: number;
+      file_path: string;
+      mime_type: string;
+      bytes: number;
+      revised_prompt?: string;
+      url?: string;
+      b64_json?: string;
+    } = {
+      index: image.index,
+      file_path: toRelativeFilePath(options.outputBaseRoot, resolvedPath),
+      mime_type: payload.mimeType,
+      bytes: payload.buffer.length,
+    };
+    if (image.revised_prompt) {
+      entry.revised_prompt = image.revised_prompt;
+    }
+    if (options.includeData) {
+      if (image.url) entry.url = image.url;
+      if (image.b64_json) entry.b64_json = image.b64_json;
+    }
+    output.push(entry);
+  }
+  return output;
+}
+
+function decodeImagePayload(image: { url?: string; b64_json?: string }): { buffer: Buffer; mimeType: string } | null {
+  if (image.b64_json) {
+    const mimeType = extractMimeFromDataUrl(image.url) ?? "image/png";
+    return { buffer: Buffer.from(image.b64_json, "base64"), mimeType };
+  }
+  if (image.url?.startsWith("data:")) {
+    const match = image.url.match(/^data:([^;]+);base64,(.+)$/i);
+    if (!match) {
+      return null;
+    }
+    return { buffer: Buffer.from(match[2], "base64"), mimeType: match[1] };
+  }
+  return null;
+}
+
+function extractMimeFromDataUrl(url?: string): string | null {
+  if (!url?.startsWith("data:")) {
+    return null;
+  }
+  const match = url.match(/^data:([^;]+);base64,/i);
+  return match ? match[1] : null;
+}
+
+function extensionForMime(mimeType: string): string {
+  switch (mimeType.toLowerCase()) {
+    case "image/jpeg":
+      return "jpg";
+    case "image/webp":
+      return "webp";
+    case "image/gif":
+      return "gif";
+    default:
+      return "png";
+  }
+}
+
+function resolveOutputPath(
+  index: number,
+  created: number,
+  extension: string,
+  options: FileMaterializeOptions
+): string {
+  return path.join(options.outputDir, `image-${created}-${index}.${extension}`);
+}
+
+function toRelativeFilePath(outputBaseRoot: string, resolvedPath: string): string {
+  const relative = path.relative(outputBaseRoot, resolvedPath);
+  if (!relative || relative === "") {
+    return ".";
+  }
+  return relative.split(path.sep).join("/");
+}
+
+async function resolveOptionalImageInputToUrl(input: {
+  image_path?: string;
+  image_url?: string;
+}): Promise<string | undefined> {
+  if (input.image_path) {
+    return imageDataUrlFromPath(input.image_path);
+  }
+  if (!input.image_url) {
+    return undefined;
+  }
+  if (
+    input.image_url.startsWith("data:image/") ||
+    input.image_url.startsWith("http://") ||
+    input.image_url.startsWith("https://")
+  ) {
+    return input.image_url;
+  }
+  throw typedError(
+    "invalid_request",
+    "image_url must be an http(s) URL or data:image/* URL."
+  );
+}
+
+function buildGenerateImageSummary(count: number): string {
+  return count === 1 ? "Generated 1 image file." : `Generated ${count} image files.`;
+}
+
+function buildGenerateImageTextPayload(output: {
+  ok: boolean;
+  summary: string;
+  model: string;
+  file_path?: string;
+  file_paths?: string[];
+}): Record<string, unknown> {
+  return {
+    ok: output.ok,
+    summary: output.summary,
+    ...(output.file_path ? { file_path: output.file_path } : {}),
+    ...(output.file_paths ? { file_paths: output.file_paths } : {}),
+    model: output.model,
+  };
+}