multimodal-mcp 0.2.0__py3-none-any.whl

multimodal_mcp/__init__.py

@@ -0,0 +1,11 @@
+import argparse
+
+from .config import load
+from .server import build
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(prog="multimodal-mcp")
+    parser.add_argument("--config", help="Config file (default: ~/.agents/mcp/multimodal/config.toml)")
+    args = parser.parse_args()
+    build(load(args.config)).run(show_banner=False)

multimodal_mcp/catalog/__init__.py

@@ -0,0 +1,18 @@
+"""Model data looked up by endpoint and model ID: API notes and default capabilities in api-notes.toml, prompt guides
+in prompt-guides.toml. Each file's header states its scope; data of another kind goes in a file of its own."""
+
+import re
+
+# A trailing date version: -YYMMDD on Ark, -YYYY-MM-DD on OpenAI snapshots.
+DATE = re.compile(r"-(\d{6}|\d{4}-\d{2}-\d{2})$")
+
+
+def notes_name(model: str) -> str:
+    """Catalog names compare case-insensitively, after the last "/" of a repo ID or path."""
+    return model.rsplit("/", 1)[-1].lower()
+
+
+def matching_names(model: str) -> set[str]:
+    """A model ID's names to match: with and without a trailing date version."""
+    name = notes_name(model)
+    return {name, DATE.sub("", name)}

multimodal_mcp/catalog/api-notes.toml

@@ -0,0 +1,293 @@
+# API notes for the configured models, which get_model_guide returns: what each call accepts and does. Notes cover
+# parameter values and ranges, required and unsupported parameters, input formats, sizes, counts and durations, inputs
+# that cannot be combined, options fields, output formats, and side effects such as billing or retention; a character
+# or token limit is an API note. How to write a prompt (structure, reference labels, markup, wording, length)
+# belongs in prompt-guides.toml and the vendor references it lists.
+# [<endpoint>] notes apply to every model on that endpoint. [<endpoint>.models.<name>] notes apply to one model, or
+# to a family whose models share them: <name> is the vendor's standard API name without its date version, a
+# self-hosted model's Hugging Face repo ID, or a glob over a family's names, and aliases lists the model's other
+# official names. capabilities lists what the model can do on the endpoint, which a configured model uses unless it
+# lists its own.
+# A configured model takes the first entry whose <name> or an alias matches it as a glob, ignoring case and anything
+# up to the last "/", with or without a trailing date version (-YYMMDD or -YYYY-MM-DD).
+# Notes are keyed by a tool, or by a category when shared across that category's tools. Each note's comment, and the
+# comment on capabilities, gives its source: a doc URL and section, source code at a pinned version, or the observed error.
+
+[ark]
+image = [
+  "Input images: jpeg, png, webp, bmp, tiff, gif, heic or heif, each up to 30 MB.",  # https://www.volcengine.com/docs/82379/1541523 §image: requirements for each input image
+  "quality is unsupported.",  # src/multimodal_mcp/endpoints/ark.py Ark.generate_image
+]
+asr = [
+  "language, prompt, hotwords and speakers are unsupported.",  # src/multimodal_mcp/endpoints/ark.py Ark.transcribe
+]
+generate_video = [
+  "Frames and references cannot be combined in one request.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.role: first frame, first and last frames, and all-modal reference are three mutually exclusive scenarios
+  "size is unsupported; use aspect_ratio and resolution.",  # src/multimodal_mcp/endpoints/ark.py Ark.generate_video
+  "Reference videos must be public URLs or asset:// IDs.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.video_url.url: a video URL or an asset ID; observed error: InvalidParameter ... reference_video must be provided as a web url
+  "Requests are capped at 64 MB and local files are sent as base64, so pass large media as URLs or asset:// IDs.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.image_url.url, §content.audio_url.url: request body at most 64 MB, no base64 for large files
+]
+
+[ark.models.doubao-seedance-2-0]
+aliases = ["doubao-seedance-2.0"]  # https://www.volcengine.com/docs/82379/2516288 Agent Plan model list
+capabilities = ["text", "frames", "reference"]  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.role: text, first frame, first and last frames, all-modal reference
+generate_video = [
+  "Frames: first_frame alone or with last_frame.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.role: first frame, or first and last frames
+  "References: up to 9 images, 3 videos and 3 audios; audio needs at least one reference image or video.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.image_url.url, §content.video_url.url, §content.audio_url.url: Seedance 2.0 series limits; audio cannot be the only reference
+  "Each reference video or audio runs 2-15 s, with at most 15 s in total per type.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.video_url.url, §content.audio_url.url: Seedance 2.0 series durations
+  "Reference images and videos showing real human faces must be this account's Seedance outputs from the last 30 days, asset:// virtual avatars or authorized material.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content: permitted sources for real human faces in reference images and videos
+  "seconds: 4-15, or -1 to let the model choose.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §duration
+  "resolution: 480p, 720p (default), 1080p or 4k; aspect_ratio: 16:9, 4:3, 1:1, 3:4, 9:16, 21:9 or adaptive (default).",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §resolution, §ratio
+  "Sound is generated by default; options {\"generate_audio\": false} makes the video silent.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §generate_audio
+]
+
+[ark.models.doubao-seedance-2-5]
+capabilities = ["text", "frames", "reference"]  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.role: text, first frame, first and last frames, all-modal reference
+generate_video = [
+  "Frames: first_frame alone or with last_frame; a last_frame of another aspect ratio is cropped to the first frame's.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.role: first frame, first and last frames
+  "References: up to 30 images, 10 videos and 10 audios, at most 50 in all; audio may be the only reference.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.image_url.url, §content.video_url.url, §content.audio_url.url: Seedance 2.5 limits; sd25-pe §Asset Specification Preflight: 50 assets in all
+  "Each reference video runs 2-30 s, or 4-30 s for a video to edit, and each reference audio 2-30 s, with at most 30 s in total per type.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content.video_url.url, §content.audio_url.url: Seedance 2.5 durations
+  "Reference images and videos showing real human faces must be this account's Seedance outputs from the last 30 days, asset:// virtual avatars or authorized material.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §content: permitted sources for real human faces in reference images and videos
+  "seconds: 4-30, or -1 (default) to let the model choose; resolution: 480p, 720p (default) or 1080p; aspect_ratio: 16:9, 4:3, 1:1, 3:4, 9:16, 21:9 or adaptive (default).",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §duration, §resolution, §ratio
+  "With frames the aspect ratio follows first_frame. Editing or extending a reference video keeps its aspect ratio, and editing also keeps its length, up to about 0.4 s shorter; these tasks take no aspect_ratio, and editing takes no seconds.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §ratio: Seedance 2.5 adaptive only for editing, extension and frames; §duration: editing only -1
+  "The model infers whether a reference task generates, edits or extends from the media and prompt, and fails the job when the parameters do not fit that task; options {\"omni_reference_task_type\": \"reference\"}, \"edit\" or \"extend\" checks that task's limits on submission instead.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §omni_reference_task_type: InvalidParameter.TaskTypeConstraint, TaskTypeMismatch
+  "Sound is generated by default; options {\"generate_audio\": false} makes the video silent.",  # https://ark.volcengine.com/region:cn-beijing/docs/ark/create-video-generation-task-api §generate_audio
+]
+
+[ark.models.doubao-seedream-5-0-pro]
+aliases = ["doubao-seedream-5.0-pro"]  # observed: accepted by the Ark API
+capabilities = ["generate", "edit", "reference"]  # https://www.volcengine.com/docs/82379/1541523 §image; src/multimodal_mcp/endpoints/ark.py Ark.generate_image rejects mask
+image = [
+  "size: 1K, 1.5K or 2K (default) with the aspect ratio described in the prompt, or WIDTHxHEIGHT with 921,600-4,624,220 pixels and an aspect ratio within 1:16-16:1.",  # https://www.volcengine.com/docs/82379/1541523 §size: Seedream 5.0 pro / Seedream 5.0 flash
+  "Up to 10 input images.",  # https://www.volcengine.com/docs/82379/1541523 §image: Seedream 5.0 pro / Seedream 5.0 flash take up to 10
+]
+
+[ark.models."doubao-seed-tts-2.0"]
+aliases = ["seed-tts-2.0"]  # https://www.volcengine.com/docs/82379/2516286#aa1f3c9e core configuration: doubao-seed-tts-2.0 uses X-Api-Resource-Id seed-tts-2.0
+capabilities = ["voice", "timestamps"]  # https://www.volcengine.com/docs/6561/1598757 §req_params.speaker, §req_params.audio_params.enable_subtitle; src/multimodal_mcp/endpoints/ark.py Ark.synthesize rejects reference_audio, and Ark has no voice design
+generate_speech = [
+  "voice must be a Doubao TTS 2.0 voice such as zh_female_vv_uranus_bigtts; voices are listed at https://www.volcengine.com/docs/6561/1257544.",  # https://www.volcengine.com/docs/6561/1598757 §X-Api-Resource-Id: seed-tts-2.0 supports only Doubao TTS 2.0 voices
+  "speed: 0.5-2.0.",  # https://www.volcengine.com/docs/6561/1598757 §req_params.audio_params.speech_rate: [-50, 100] maps to 0.5x-2.0x
+  "response_format: mp3 (default), ogg_opus or pcm.",  # https://www.volcengine.com/docs/6561/1598757 §req_params.audio_params.format
+  "Timestamps cover Chinese and English only.",  # https://www.volcengine.com/docs/6561/1598757 §req_params.audio_params.enable_subtitle
+  "instructions is sent as the one entry of context_texts, which holds either a voice instruction or the line said before input.",  # src/multimodal_mcp/endpoints/ark.py Ark.synthesize: additions.context_texts = [instructions]; https://www.volcengine.com/docs/6561/1598757 §req_params.additions.context_texts
+]
+
+[ark.models."doubao-seed-asr-2.0"]
+capabilities = ["text", "segment", "word"]  # src/multimodal_mcp/endpoints/ark.py Ark.transcribe: utterances with words; rejects speakers and hotwords
+
+[minimax]
+image = [
+  "Prompt up to 1,500 characters.",  # https://platform.minimaxi.com/docs/api-reference/image-generation-t2i §prompt
+  "references are portrait subjects, as JPG or PNG under 10 MB.",  # https://platform.minimaxi.com/docs/api-reference/image-generation-i2i §subject_reference
+  "quality is unsupported.",  # src/multimodal_mcp/endpoints/minimax.py MiniMax.generate_image
+]
+generate_video = [
+  "Frames and references cannot be combined in one request.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §content: image-to-video and multimodal reference-to-video are mutually exclusive
+  "size is unsupported; use aspect_ratio and resolution.",  # src/multimodal_mcp/endpoints/minimax.py MiniMax.generate_video
+  "resolution and seconds are required.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §VideoGenerationV2Req.required
+  "Text-to-video needs aspect_ratio 21:9, 16:9, 4:3, 1:1, 3:4 or 9:16; with frames the ratio follows the image.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §ratio
+  "References: up to 9 images, 3 videos and 3 audios; each video or audio runs 2-15 s, with at most 15 s in total per type.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §content: input media limits
+  "Media may be public URLs or mm_file:// IDs; requests are capped at 64 MB and local files are sent as base64.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §content: input media limits; §image_url.url
+  "Prompt up to 7,000 characters.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §content.text
+]
+generate_speech = [
+  "input under 10,000 characters.",  # https://platform.minimaxi.com/docs/api-reference/speech-t2a-http §text
+  "instructions is unsupported; for emotion, pass options {\"voice_setting\": {\"emotion\": \"sad\"}} with happy, sad, angry, fearful, disgusted, surprised or calm, which by default the model picks from the text; speech-2.6 models also take fluent and whisper.",  # https://platform.minimaxi.com/docs/api-reference/speech-t2a-http §voice_setting.emotion
+  "options also takes pronunciation_dict, such as {\"tone\": [\"处理/(chu3)(li3)\", \"omg/oh my god\"]}; text_normalization_mode basic or quality, off by default; and language_boost, such as \"Chinese,Yue\" or \"auto\".",  # https://platform.minimax.io/docs/api-reference/speech-t2a-http §pronunciation_dict, §text_normalization_mode, §language_boost
+  "voice: a system voice such as English_Graceful_Lady or Chinese (Mandarin)_Lyrical_Voice, or a cloned or designed voice ID.",  # https://platform.minimaxi.com/docs/api-reference/speech-t2a-http §voice_setting.voice_id
+  "speed: 0.5-2.",  # https://platform.minimaxi.com/docs/api-reference/speech-t2a-http §voice_setting.speed
+  "response_format: mp3 (default), pcm, flac, wav, pcmu_raw, pcmu_wav or opus.",  # https://platform.minimaxi.com/docs/api-reference/speech-t2a-http §audio_setting.format
+]
+voice_design = [
+  "preview_text is required, up to 500 characters; the preview is billed.",  # https://platform.minimaxi.com/docs/api-reference/voice-design-design §preview_text
+]
+
+[minimax.models.MiniMax-H3]
+capabilities = ["text", "frames", "reference"]  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §content: text, image-to-video, multimodal reference-to-video
+generate_video = [
+  "resolution: 768P or 2K; seconds: 4-15.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §model, §resolution, §duration
+]
+
+[minimax.models.MiniMax-H3-Max]
+capabilities = ["text", "frames", "reference"]  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §content: text, image-to-video, multimodal reference-to-video
+generate_video = [
+  "resolution: 480P or 768P; seconds: 5-15.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §model, §resolution, §duration
+  "Prompt expansion defaults to balanced; options {\"extra\": {\"prompt_expansion_mode\": \"disabled\"}} turns it off, and \"quality\" expands more.",  # https://platform.minimaxi.com/docs/api-reference/video-generation-v2-create §extra.prompt_expansion_mode
+]
+
+[minimax.models.image-01]
+capabilities = ["generate", "reference"]  # https://platform.minimaxi.com/docs/api-reference/image-generation-i2i §subject_reference; src/multimodal_mcp/endpoints/minimax.py MiniMax.generate_image rejects mask
+image = [
+  "size: 1:1 (default), 16:9, 4:3, 3:2, 2:3, 3:4, 9:16 or 21:9, or WIDTHxHEIGHT with both sides 512-2048 in multiples of 8.",  # https://platform.minimaxi.com/docs/api-reference/image-generation-t2i §aspect_ratio, §width
+  "options {\"prompt_optimizer\": true} lets the service expand the prompt; it is off by default.",  # https://platform.minimax.io/docs/api-reference/image-generation-t2i §prompt_optimizer
+  "For a consistent character, pass one frontal photo of a single person as the reference.",  # https://platform.minimaxi.com/docs/api-reference/image-generation-i2i §subject_reference
+]
+
+[minimax.models."asr-1.0"]
+capabilities = ["text", "segment", "word", "speaker"]  # https://platform.minimaxi.com/docs/api-reference/speech-to-text §timestamp_level; src/multimodal_mcp/endpoints/minimax.py MiniMax.transcribe: verbose_json carries timestamps and speaker labels
+asr = [
+  "prompt and hotwords are unsupported.",  # src/multimodal_mcp/endpoints/minimax.py MiniMax.transcribe; https://platform.minimaxi.com/docs/api-reference/speech-to-text has no such fields
+  "language: a BCP-47 tag such as en-US; omit it for mixed-language audio.",  # https://platform.minimaxi.com/docs/api-reference/speech-to-text §language (header)
+  "Audio up to 500 s and 50 MB in wav, aiff, flac, m4a, mp3, aac, opus or ogg.",  # https://platform.minimaxi.com/docs/api-reference/speech-to-text §file
+]
+transcribe_audio_with_timestamps = [
+  "Word timestamps are per character for Chinese.",  # https://platform.minimaxi.com/docs/api-reference/speech-to-text §timestamp_level
+]
+
+[minimax.models."speech-2.8-hd"]
+capabilities = ["voice", "timestamps", "design"]  # https://platform.minimaxi.com/docs/api-reference/speech-t2a-http §voice_setting.voice_id, §subtitle_enable; https://platform.minimaxi.com/docs/api-reference/voice-design-design
+
+[minimax.models."speech-2.8-turbo"]
+capabilities = ["voice", "timestamps", "design"]  # https://platform.minimaxi.com/docs/api-reference/speech-t2a-http §voice_setting.voice_id, §subtitle_enable; https://platform.minimaxi.com/docs/api-reference/voice-design-design
+
+[gemini]
+image = [
+  "quality is unsupported.",  # src/multimodal_mcp/endpoints/gemini.py Gemini.generate_image
+]
+generate_video = [
+  "size is unsupported; use aspect_ratio and resolution.",  # src/multimodal_mcp/endpoints/gemini.py Gemini.generate_video
+]
+
+[gemini.models."gemini-omni-1.1-flash"]
+capabilities = ["text", "frames", "reference"]  # https://ai.google.dev/gemini-api/docs/omni §Using tags in prompts to set image and video roles
+generate_video = [
+  "aspect_ratio: 16:9 (default) or 9:16; resolution: 360p, 720p (default), 1080p or 4k.",  # https://ai.google.dev/gemini-api/docs/omni §Control aspect ratio, §Output resolution
+  "seconds is unsupported; clips run 3-10 s.",  # src/multimodal_mcp/endpoints/gemini.py Gemini.generate_video; https://ai.google.dev/gemini-api/docs/models/gemini-omni-flash §Limits
+  "last_frame needs first_frame.",  # https://ai.google.dev/gemini-api/docs/omni §Using tags in prompts to set image and video roles
+  "The tool writes the [# Sources ...] and [# References ...] declarations before the prompt, binding the frames and references to their tags.",  # src/multimodal_mcp/endpoints/gemini.py Gemini.generate_video: declaration + prompt
+  "Reference videos: up to 3 clips of up to 3 s each, with their audio ignored; reference audio is unsupported.",  # https://ai.google.dev/gemini-api/docs/omni §Limitations
+  "Recognizable people in uploaded images are unsupported.",  # https://ai.google.dev/gemini-api/docs/omni §Limitations
+  "A video to edit or extend runs up to 10 s; extension only appends to its end, and voices cannot be edited.",  # https://ai.google.dev/gemini-api/docs/omni §Limitations
+]
+
+[gemini.models."gemini-3.1-flash-image"]
+capabilities = ["generate", "edit", "reference"]  # https://ai.google.dev/gemini-api/docs/image-generation §Use up to 14 reference images; src/multimodal_mcp/endpoints/gemini.py Gemini.generate_image rejects mask
+image = [
+  "size: an aspect ratio (1:1, 1:4, 1:8, 2:3, 3:2, 3:4, 4:1, 4:3, 4:5, 5:4, 8:1, 9:16, 16:9 or 21:9) or a tier 512, 1K (default), 2K or 4K with uppercase K; for both, give the ratio as size and options {\"response_format\": {\"image_size\": \"2K\"}}.",  # https://ai.google.dev/gemini-api/docs/image-generation §Aspect ratios and image size (3.1 Flash Image), §Generate images up to 4K resolution; https://ai.google.dev/api/interactions-api §ImageResponseFormat.image_size
+  "Without size, output matches the input image, or is 1:1 without one.",  # https://ai.google.dev/gemini-api/docs/image-generation §Aspect ratios and image size
+  "Up to 10 object images and 4 character images.",  # https://ai.google.dev/gemini-api/docs/image-generation §Use up to 14 reference images
+]
+
+[gemini.models."gemini-3.5-transcribe"]
+capabilities = ["text", "word", "speaker", "hotwords"]  # https://ai.google.dev/gemini-api/docs/transcribe §Parameter reference; src/multimodal_mcp/endpoints/gemini.py Gemini.transcribe returns word timestamps only
+asr = [
+  "prompt is unsupported; hotwords are sent as custom_vocabulary.",  # https://ai.google.dev/gemini-api/docs/transcribe §Parameter reference; src/multimodal_mcp/endpoints/gemini.py Gemini.transcribe
+  "language: a BCP-47 code such as en-US; omit it to auto-detect, including mixed languages.",  # https://ai.google.dev/gemini-api/docs/transcribe §Parameter reference
+  "Files are sent inline, up to 100 MB per request.",  # https://ai.google.dev/gemini-api/docs/file-input-methods §Method comparison
+]
+transcribe_audio = [
+  "Audio up to 1 hour.",  # https://ai.google.dev/gemini-api/docs/transcribe §Limitations
+  "hotwords: up to 1,000 terms.",  # https://ai.google.dev/gemini-api/docs/transcribe §Limitations
+]
+transcribe_audio_with_timestamps = [
+  "Audio up to 30 minutes.",  # https://ai.google.dev/gemini-api/docs/transcribe §Limitations
+  "hotwords cannot be combined with timestamps or speakers.",  # src/multimodal_mcp/endpoints/gemini.py Gemini.transcribe; https://ai.google.dev/gemini-api/docs/transcribe §Limitations
+  "Up to 8 speakers; 3 or more is experimental.",  # https://ai.google.dev/gemini-api/docs/transcribe §Limitations
+]
+
+[gemini.models."gemini-3.8-flash-tts"]
+capabilities = ["voice", "design"]  # https://ai.google.dev/gemini-api/docs/speech-generation §Voice options; https://ai.google.dev/gemini-api/docs/voice-design; src/multimodal_mcp/endpoints/gemini.py Gemini.synthesize rejects reference_audio and timestamps
+generate_speech = [
+  "voice: a prebuilt voice such as Kore, Puck or Charon, a Voice Library voice, or a voice_ ID from voice_design.",  # https://ai.google.dev/gemini-api/docs/speech-generation §Voice options
+  "response_format: wav (default), l16 (raw PCM), mulaw or alaw.",  # https://ai.google.dev/gemini-api/docs/speech-generation §Audio output formats
+  "speed is unsupported; ask for the pace in instructions.",  # src/multimodal_mcp/endpoints/gemini.py Gemini.synthesize
+  "reference_audio and timestamps are unsupported.",  # src/multimodal_mcp/endpoints/gemini.py Gemini.synthesize; capabilities above
+  "Each call speaks one turn in one voice.",  # src/multimodal_mcp/endpoints/gemini.py Gemini.synthesize: one voice per request
+]
+voice_design = [
+  "The preview is generated from the description, and Gemini assigns the voice ID; preview_text and voice_id are unsupported.",  # https://ai.google.dev/gemini-api/docs/voice-design §How Voice design works; src/multimodal_mcp/endpoints/gemini.py Gemini.design_voice
+  "Voices are kept for a year after last use, up to 200 per project.",  # https://ai.google.dev/gemini-api/docs/voice-design §Manage your voices
+]
+
+[openai]
+transcribe_audio_with_timestamps = [
+  "speakers=true cannot be combined with granularity word; speaker labels come with segment timestamps.",  # src/multimodal_mcp/endpoints/openai.py OpenAI.transcribe: diarized_json has no word timestamps
+]
+
+[openai.models."gpt-image-2.5-sunburst"]
+capabilities = ["generate", "edit", "reference", "mask"]  # openai-python 3.28.0 src/openai/resources/images.py Images.generate, Images.edit §image, §mask
+image = [
+  "size: auto (default) or WIDTHxHEIGHT with both sides multiples of 16, aspect ratio 1:3-3:1, up to 3840x2160; above 2560x1440 is experimental.",  # openai-python 3.28.0 src/openai/resources/images.py Images.generate, Images.edit §size
+  "quality: low, medium, high, xhigh, max or auto (default).",  # openai-python 3.28.0 src/openai/resources/images.py Images.generate, Images.edit §quality
+  "Up to 16 input images, each png, webp or jpg under 50 MB.",  # openai-python 3.28.0 src/openai/resources/images.py Images.edit §image
+  "For a transparent background, pass options {\"background\": \"transparent\", \"output_format\": \"png\"}.",  # openai-python 3.28.0 src/openai/resources/images.py Images.generate, Images.edit §background
+]
+edit_image = [
+  "The mask is a PNG under 4 MB with the first image's dimensions, transparent where it should change.",  # openai-python 3.28.0 src/openai/resources/images.py Images.edit §mask
+]
+
+[openai.models."gpt-image-2.5-flare"]
+capabilities = ["generate", "edit", "reference", "mask"]  # openai-python 3.28.0 src/openai/resources/images.py Images.generate, Images.edit §image, §mask
+image = [
+  "size: auto (default) or WIDTHxHEIGHT with both sides multiples of 16, aspect ratio 1:3-3:1, up to 3840x2160; above 2560x1440 is experimental.",  # openai-python 3.28.0 src/openai/resources/images.py Images.generate, Images.edit §size
+  "quality: low, medium, high, xhigh, max or auto (default).",  # openai-python 3.28.0 src/openai/resources/images.py Images.generate, Images.edit §quality
+  "Up to 16 input images, each png, webp or jpg under 50 MB.",  # openai-python 3.28.0 src/openai/resources/images.py Images.edit §image
+  "For a transparent background, pass options {\"background\": \"transparent\", \"output_format\": \"png\"}.",  # openai-python 3.28.0 src/openai/resources/images.py Images.generate, Images.edit §background
+]
+edit_image = [
+  "The mask is a PNG under 4 MB with the first image's dimensions, transparent where it should change.",  # openai-python 3.28.0 src/openai/resources/images.py Images.edit §mask
+]
+
+[openai.models."whisper-1"]
+capabilities = ["text", "segment", "word"]  # openai-python 3.28.0 src/openai/resources/audio/transcriptions.py Transcriptions.create §timestamp_granularities
+asr = [
+  "Files up to 25 MB in flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav or webm.",  # https://developers.openai.com/api/docs/guides/speech-to-text §File transcription; openai-python 3.28.0 src/openai/resources/audio/transcriptions.py Transcriptions.create §file
+  "language: an ISO-639-1 code such as en.",  # openai-python 3.28.0 src/openai/resources/audio/transcriptions.py Transcriptions.create §language
+  "prompt: up to 224 tokens of sample text, whose spelling, punctuation and style the transcript follows.",  # https://developers.openai.com/api/docs/guides/speech-to-text §Prompting; openai-python 3.28.0 src/openai/resources/audio/transcriptions.py Transcriptions.create §prompt
+]
+
+[openai.models."gemini-*-flash"]
+capabilities = ["image", "audio"]  # https://aistudio.google.com/docs/openai §Image understanding (image_url), §Audio understanding (input_audio)
+
+[openai.models."gemini-*-argon"]
+capabilities = ["image", "audio"]  # https://aistudio.google.com/docs/openai §Image understanding (image_url), §Audio understanding (input_audio)
+
+# Self-hosted models served by vLLM or vLLM-Omni.
+
+[openai.models."Qwen/Qwen-Image-2.1"]
+capabilities = ["generate", "edit", "reference"]  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/images.py; https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/api_server.py /v1/images/edits takes a list of images; src/multimodal_mcp/endpoints/openai.py OpenAI.generate_image sends references and edit images alike to images.edit
+image = [
+  "size is WIDTHxHEIGHT.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/images.py §validate_size
+  "options takes negative_prompt, seed, num_inference_steps (1-200), guidance_scale (0-20) and true_cfg_scale (0-20).",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/images.py §ImageGenerationRequest
+]
+
+[openai.models."OpenMOSS-Team/MOSS-Transcribe-Diarize"]
+capabilities = ["text", "segment", "speaker"]  # https://github.com/vllm-project/vllm/blob/7853700d4cdc2b84c728c840114ed42cda7454a7/vllm/model_executor/models/moss_transcribe_diarize.py §parse_verbose_transcript, §parse_diarized_transcript
+asr = [
+  "prompt is the transcription instruction and replaces the default one, which asks for [start][S01]text[end] segments; hotwords is unsupported.",  # https://github.com/vllm-project/vllm/blob/7853700d4cdc2b84c728c840114ed42cda7454a7/vllm/model_executor/models/moss_transcribe_diarize.py §DEFAULT_MOSS_TRANSCRIBE_DIARIZE_PROMPT, §get_generation_prompt: only request_prompt reaches the model
+]
+transcribe_audio = [
+  "The transcript marks each segment as [start][S01]text[end]; for clean speaker segments, use transcribe_audio_with_timestamps with speakers=true.",  # https://github.com/vllm-project/vllm/blob/7853700d4cdc2b84c728c840114ed42cda7454a7/vllm/model_executor/models/moss_transcribe_diarize.py §parse_diarized_transcript
+]
+transcribe_audio_with_timestamps = [
+  "Timed segments and speaker labels are parsed from the default [start][S01]text[end] format, so a prompt for this tool must keep that format.",  # https://github.com/vllm-project/vllm/blob/7853700d4cdc2b84c728c840114ed42cda7454a7/vllm/model_executor/models/moss_transcribe_diarize.py §parse_verbose_transcript, §parse_diarized_transcript
+  "Without speakers=true, segment text keeps its [S01] speaker prefix.",  # https://github.com/vllm-project/vllm/blob/7853700d4cdc2b84c728c840114ed42cda7454a7/vllm/model_executor/models/moss_transcribe_diarize.py §parse_verbose_transcript
+]
+
+[openai.models."IndexTeam/IndexTTS-2"]
+capabilities = ["voice", "reference"]  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py: ref_audio, or a voice in uploaded_speakers; https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/api_server.py POST/GET /v1/audio/voices
+generate_speech = [
+  "Pass reference_audio to clone a voice from a local audio file, or pass voice only when the user names a speaker already uploaded to the server (there are no built-in preset voice names).",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py; https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/api_server.py /v1/audio/voices
+  "instructions is unused; for emotion, pass options {\"extra_params\": {...}} with use_emo_text true (the emotion comes from emo_text, or from input without it), or emo_vector, eight intensities 0-1.2 in the order happy, angry, sad, afraid, disgusted, melancholic, surprised, calm; emo_alpha scales either.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py §_INDEXTTS2_EMOTION_KEYS; vllm_omni/model_executor/models/indextts2/indextts2_talker.py; https://github.com/index-tts/index-tts README §Python API 3-6: emo_vector order
+  "response_format: mp3 (default), wav, pcm, flac or opus; speed: 0.25-4.0.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/audio.py §OpenAICreateSpeechRequest: formats and speed range; the openai adapter requests mp3 when response_format is unset
+]
+
+[openai.models."IndexTeam/IndexTTS-2.5"]
+capabilities = ["voice", "reference"]  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py IndexTTS25Adapter extends IndexTTS2Adapter: ref_audio, or a voice in uploaded_speakers
+generate_speech = [
+  "Pass reference_audio to clone a voice from a local audio file, or pass voice only when the user names a speaker already uploaded to the server (there are no built-in preset voice names).",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py IndexTTS2Adapter.validate; https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/api_server.py /v1/audio/voices
+  "instructions is unused; for emotion, pass options {\"extra_params\": {...}} with use_emo_text true (the emotion comes from emo_text, or from input without it), or emo_vector, eight intensities 0-1.2 in the order happy, angry, sad, afraid, disgusted, melancholic, surprised, calm; emo_alpha scales either.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py §_INDEXTTS2_EMOTION_KEYS, inherited by IndexTTS25Adapter; vllm_omni/model_executor/models/indextts2/indextts2_talker.py
+  "extra_params also takes lang, the text's language code, zh by default (such as en, ja, yue, or zhen for mixed Chinese and English), and text_normalization, a boolean that defaults to true.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py IndexTTS25Adapter._validate_extra_params, _build_params; https://github.com/vllm-project/vllm-omni/blob/88cd44ad3624b16ef122049bd6534dd4e1e19195/vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py language codes
+  "response_format: mp3 (default), wav, pcm, flac or opus; speed: 0.5-2.0, which scales the spoken duration (2.0 speaks twice as fast).",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/audio.py §OpenAICreateSpeechRequest: formats; https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py IndexTTS25Adapter.validate: speed 0.5-2.0, duration_factor = 1 / speed; the openai adapter requests mp3 when response_format is unset
+]
+
+[openai.models."MiniMaxAI/MiniMax-H3"]
+capabilities = ["text", "frames", "reference"]  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/videos.py
+generate_video = [
+  "Text-to-video needs aspect_ratio 21:9, 16:9, 4:3, 1:1, 3:4 or 9:16; with frames the ratio follows the image, and with references it defaults to 16:9.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/videos.py §aspect_ratio
+  "Reference videos and audios run 2-15 s each.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/model_executor/models/minimax_h3/reference_video.py
+  "resolution is unsupported; use aspect_ratio or size.",  # src/multimodal_mcp/endpoints/openai.py OpenAI.generate_video
+  "seconds is a positive whole number.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/videos.py §SecondStr
+  "Fields beyond the tool parameters go in options under extra_params.",  # https://github.com/vllm-project/vllm-omni/blob/64903b8e30b6595e4018c66232abbbd806aa9dfd/vllm_omni/entrypoints/openai/protocol/videos.py §extra_params
+]

multimodal_mcp/catalog/api_notes.py

@@ -0,0 +1,34 @@
+"""The API notes of api-notes.toml, whose header describes their scope and layout."""
+
+import fnmatch
+import tomllib
+from pathlib import Path
+
+from . import matching_names, notes_name
+
+API_NOTES = tomllib.loads((Path(__file__).parent / "api-notes.toml").read_text(encoding="utf-8"))
+
+
+def model_entry(endpoint: str, model: str) -> dict:
+    """The model's entry under the endpoint, the first whose name or an alias matches it as a glob; empty when it has
+    none."""
+    matching = matching_names(model)
+    return next(
+        (
+            entry
+            for key, entry in API_NOTES.get(endpoint, {}).get("models", {}).items()
+            if any(
+                fnmatch.fnmatchcase(name, notes_name(glob))
+                for name in matching
+                for glob in (key, *entry.get("aliases", []))
+            )
+        ),
+        {},
+    )
+
+
+def rules(endpoint: str, model: str, category: str, tool: str) -> list[str]:
+    """The API notes for a tool on this model: the endpoint's, then the model's, each by category, then by tool."""
+    table = API_NOTES.get(endpoint, {})
+    entry = model_entry(endpoint, model)
+    return [rule for scope in (table, entry) for key in (category, tool) for rule in scope.get(key, [])]