multimodal-mcp 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multimodal_mcp-0.2.0/LICENSE +21 -0
- multimodal_mcp-0.2.0/PKG-INFO +168 -0
- multimodal_mcp-0.2.0/README.md +148 -0
- multimodal_mcp-0.2.0/pyproject.toml +35 -0
- multimodal_mcp-0.2.0/pyproject.toml.orig +36 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/__init__.py +11 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/catalog/__init__.py +18 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/catalog/api-notes.toml +293 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/catalog/api_notes.py +34 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/catalog/prompt-guides.toml +247 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/catalog/prompt_guides.py +281 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/config.py +98 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/endpoints/__init__.py +84 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/endpoints/ark.py +205 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/endpoints/gemini.py +142 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/endpoints/minimax.py +131 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/endpoints/openai.py +107 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/omni/__init__.py +5 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/omni/agents.py +367 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/omni/instructions.md +51 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/omni/media.py +87 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/omni/workspace.py +326 -0
- multimodal_mcp-0.2.0/src/multimodal_mcp/server.py +649 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vigilans
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: multimodal-mcp
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: MCP server for image, audio and video generation, transcription, voice design and media analysis.
|
|
5
|
+
Author: Vigilans
|
|
6
|
+
Author-email: Vigilans <vigilans@foxmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: fastmcp>=4.1.0
|
|
10
|
+
Requires-Dist: httpx>=0.28.1
|
|
11
|
+
Requires-Dist: openai>=3.28.0
|
|
12
|
+
Requires-Dist: pydantic>=2.14.0
|
|
13
|
+
Requires-Dist: pydantic-ai-harness>=0.66
|
|
14
|
+
Requires-Dist: pydantic-ai-slim[openai]>=2.66
|
|
15
|
+
Requires-Dist: static-ffmpeg>=3.0
|
|
16
|
+
Requires-Dist: websockets>=15
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Project-URL: Repository, https://github.com/Vigilans/multimodal-mcp
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# multimodal-mcp
|
|
22
|
+
|
|
23
|
+
One MCP server for image generation and editing, speech recognition, speech synthesis and voice design, video generation, and agents that analyze images, audio and video. Declare the models and their capabilities in a config file; the server registers a tool only when a configured model supports it, and picks the model for each call from what the call needs.
|
|
24
|
+
|
|
25
|
+
## Installation
|
|
26
|
+
|
|
27
|
+
The server requires Python 3.12 or later and runs over stdio:
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
uvx multimodal-mcp
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
or install it with `pip install multimodal-mcp` and run `multimodal-mcp`. For the development version, run `uvx --from git+https://github.com/Vigilans/multimodal-mcp multimodal-mcp`.
|
|
34
|
+
|
|
35
|
+
Register it with your MCP client as a stdio server named `multimodal`, for example:
|
|
36
|
+
|
|
37
|
+
```json
|
|
38
|
+
{
|
|
39
|
+
"mcpServers": {
|
|
40
|
+
"multimodal": {
|
|
41
|
+
"command": "uvx",
|
|
42
|
+
"args": ["multimodal-mcp", "--config", "/home/me/.agents/mcp/multimodal/config.toml"]
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The skill in [skills/multimodal](skills/multimodal/SKILL.md) tells agents how to choose models, write prompts, check results and deliver them. Install it with `npx skills add Vigilans/multimodal-mcp -s multimodal`, adding `-g` to install it globally.
|
|
49
|
+
|
|
50
|
+
On Windows, omni agents run their commands in Git Bash, so they require Git for Windows with `git` on `PATH`.
|
|
51
|
+
|
|
52
|
+
## Configuration
|
|
53
|
+
|
|
54
|
+
The config file is read from `--config <path>`, then `MULTIMODAL_MCP_CONFIG`, then `~/.agents/mcp/multimodal/config.toml`.
|
|
55
|
+
|
|
56
|
+
```toml
|
|
57
|
+
[providers.gateway]
|
|
58
|
+
endpoint = "openai"
|
|
59
|
+
base_url = "https://gateway.example.com/v1"
|
|
60
|
+
env_key = "GATEWAY_API_KEY" # or: api_key = "..."
|
|
61
|
+
|
|
62
|
+
[providers.gemini]
|
|
63
|
+
endpoint = "gemini" # base_url omitted: the official API
|
|
64
|
+
env_key = "GEMINI_API_KEY"
|
|
65
|
+
|
|
66
|
+
[[models.image]] # models.<image|asr|tts|video|omni>
|
|
67
|
+
provider = "gateway"
|
|
68
|
+
model = "gpt-image-2.5-flare"
|
|
69
|
+
description = "Fast image generation and editing"
|
|
70
|
+
capabilities = ["generate", "edit", "reference", "mask"]
|
|
71
|
+
|
|
72
|
+
[[models.video]]
|
|
73
|
+
provider = "gateway"
|
|
74
|
+
model = "MiniMax-H3"
|
|
75
|
+
description = "Video with synchronized audio; text, first/last frame, or reference driven"
|
|
76
|
+
capabilities = ["text", "frames"]
|
|
77
|
+
|
|
78
|
+
[[models.tts]] # capabilities from the API notes
|
|
79
|
+
provider = "gemini"
|
|
80
|
+
model = "gemini-3.8-flash-tts"
|
|
81
|
+
|
|
82
|
+
[[models.omni]] # omni models use an openai endpoint
|
|
83
|
+
provider = "gateway"
|
|
84
|
+
model = "gemini-3.5-flash"
|
|
85
|
+
capabilities = ["image", "audio"]
|
|
86
|
+
|
|
87
|
+
[defaults] # optional, per category
|
|
88
|
+
image = "gpt-image-2.5-flare"
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
A model without `capabilities` takes the list from [api-notes.toml](src/multimodal_mcp/catalog/api-notes.toml) for its endpoint and model ID, and the config fails to load when the API notes have none; the server treats unlisted capabilities as unsupported. Capabilities in the config must match what the endpoint implements for that model, since the server sends every call the listed capabilities allow. Tools, `list_models` and `defaults` name a model by its `model` value, or by `<provider>/<model>` when several entries share that value, such as `gateway/gpt-image-2.5-flare` beside `openai/gpt-image-2.5-flare`.
|
|
92
|
+
|
|
93
|
+
### Capabilities
|
|
94
|
+
|
|
95
|
+
| Category | Capability | Meaning |
|
|
96
|
+
|---|---|---|
|
|
97
|
+
| `image` | `generate`, `edit` | Create an image from a prompt; modify given images |
|
|
98
|
+
| | `reference` | Generate a new image following a reference's subject or style |
|
|
99
|
+
| | `mask` | Edit only a masked region |
|
|
100
|
+
| `asr` | `text` | Plain transcription |
|
|
101
|
+
| | `segment`, `word` | Timestamp granularity |
|
|
102
|
+
| | `speaker` | Speaker labels |
|
|
103
|
+
| | `hotwords` | A dedicated hotword field: vLLM `hotwords`, Gemini `custom_vocabulary` |
|
|
104
|
+
| `tts` | `voice` | Speak with a preset or registered voice ID |
|
|
105
|
+
| | `reference` | Imitate a voice sample sent with the request |
|
|
106
|
+
| | `timestamps` | Return segment and word timing of the generated speech |
|
|
107
|
+
| | `design` | Create a reusable voice ID from a description |
|
|
108
|
+
| `video` | `text` | Text-only generation |
|
|
109
|
+
| | `frames` | First frame, or first and last frames |
|
|
110
|
+
| | `reference` | Reference images, videos or audio |
|
|
111
|
+
| `omni` | `image`, `audio` | The media the model reads |
|
|
112
|
+
|
|
113
|
+
## Tools and results
|
|
114
|
+
|
|
115
|
+
| Tool | Registered when |
|
|
116
|
+
|---|---|
|
|
117
|
+
| `list_models(category?, capability?)` | always |
|
|
118
|
+
| `get_model_guide(model, lang, update?, api_notes?, prompt_guide?)` | always |
|
|
119
|
+
| `generate_image` | an `image` model has `generate` |
|
|
120
|
+
| `edit_image` | an `image` model has `edit` |
|
|
121
|
+
| `transcribe_audio` | an `asr` model has `text` |
|
|
122
|
+
| `transcribe_audio_with_timestamps` | an `asr` model has `segment` or `word` |
|
|
123
|
+
| `generate_speech` | a `tts` model has `voice` or `reference` |
|
|
124
|
+
| `voice_design` | a `tts` model has `design` |
|
|
125
|
+
| `generate_video`, `get_video_job` | a `video` model exists; both list every video model |
|
|
126
|
+
| `start_omni_agent`, `continue_omni_agent`, `get_omni_agent`, `wait_omni_agents`, `stop_omni_agent` | an `omni` model exists |
|
|
127
|
+
|
|
128
|
+
`get_model_guide` returns a model's API notes from [api-notes.toml](src/multimodal_mcp/catalog/api-notes.toml) and its family's prompt guide from [prompt-guides.toml](src/multimodal_mcp/catalog/prompt-guides.toml). The vendor reference files a guide names are downloaded into `~/.cache/multimodal-mcp/references/<family>/` on first use. Existing files are reused for either `lang`; `update=true` downloads them again in the requested `lang`, and a failed download keeps the existing file and returns an error. A deleted file is downloaded again on the next call.
|
|
129
|
+
|
|
130
|
+
Each tool's `model` parameter is an enum of the models that can perform it. Call arguments add requirements: `references` needs `reference`, `mask` needs `mask`, `granularity: "word"` needs `word`, `speakers` needs `speaker`, `hotwords` needs `hotwords`, `voice` and `reference_audio` need `voice` and `reference`, `timestamps` needs `timestamps`; a video with neither frames nor references needs `text`, `first_frame` and `last_frame` need `frames`, `reference_images`, `reference_videos` and `reference_audios` need `reference`. A named model that lacks a requirement is rejected before any request. Without `model`, the server uses the only capable model, else the category default if capable, else asks for a choice. Results report the model used.
|
|
131
|
+
|
|
132
|
+
`generate_image`, `edit_image`, `transcribe_audio`, `transcribe_audio_with_timestamps`, `generate_speech` and `generate_video` take `options`, merged into the endpoint's request body, for fields beyond the common parameters; the API notes list the useful ones.
|
|
133
|
+
|
|
134
|
+
Results are written to `~/.cache/multimodal-mcp/<YYYY-MM-DD>/<category>/` unless an absolute `output_path` is given, and tools return the file path. `generate_video` submits a job; `get_video_job` saves the video when the job completes, to its own `output_path` when given. Timestamped transcription and `generate_speech(timestamps=true)` write `{text, duration?, segments: [{start, end, text, speaker?}], words: [{start, end, word, speaker?}]}` with times in seconds, where either list may be empty.
|
|
135
|
+
|
|
136
|
+
Omni agents work in the background: `start_omni_agent` returns an `agent_id` at once, `wait_omni_agents` waits until an agent's run ends, and `get_omni_agent` returns its answer and a page of the tool calls it made. `continue_omni_agent` adds a message to the active run, or starts a new run after the previous one ended; `stop_omni_agent` cancels the current run. Agents live in server memory and are lost when the server restarts, and a provider request already in flight may complete and be billed after cancellation. They read files and run shell commands such as `ffmpeg` and `ffprobe`, taken from `PATH` or, when `PATH` lacks either, downloaded once with static-ffmpeg.
|
|
137
|
+
|
|
138
|
+
## Endpoints
|
|
139
|
+
|
|
140
|
+
`base_url` is the API root to which the endpoint appends its paths. When omitted, the endpoint's official API is used.
|
|
141
|
+
|
|
142
|
+
| Endpoint | Default `base_url` | Image | ASR | TTS | Video |
|
|
143
|
+
|---|---|---|---|---|---|
|
|
144
|
+
| `openai` | `https://api.openai.com/v1` | `/images/generations`, `/images/edits` | `/audio/transcriptions` | `/audio/speech` | `/videos`, multipart, as served by vLLM-Omni |
|
|
145
|
+
| `minimax` | `https://api.minimaxi.com` | `/v1/image_generation` (references are portrait subjects) | `/v1/speech_to_text` | `/v1/t2a_v2`, `/v1/voice_design` | `/v2/video_generation` |
|
|
146
|
+
| `ark` | `https://ark.cn-beijing.volces.com/api/v3` for image and video, `https://openspeech.bytedance.com/api/v3` for speech | `/images/generations` | `/sauc/bigmodel_nostream` WebSocket; needs `ffmpeg` on `PATH` | `/tts/unidirectional` | `/contents/generations/tasks` |
|
|
147
|
+
| `gemini` | `https://generativelanguage.googleapis.com/v1beta` | `/interactions` | `/interactions` | `/interactions`, `/voices` | `/interactions`, background |
|
|
148
|
+
|
|
149
|
+
Omni models use the `openai` endpoint's chat completions.
|
|
150
|
+
|
|
151
|
+
An `ark` `base_url` replaces both default roots (media and speech), so a provider with separate image/video and speech roots needs one provider entry per root. Include any plan prefix, such as `.../api/plan/v3` for Agent Plan images and video and `.../api/v3/plan` for Agent Plan speech. Ark speech models select their service with `X-Api-Resource-Id`: `doubao-seed-tts-2.0` and `doubao-seed-asr-2.0` map to `seed-tts-2.0` and `volc.seedasr.sauc.duration`; any other model name is sent as the resource ID.
|
|
152
|
+
|
|
153
|
+
`minimax` and `ark` video inputs pass URLs, `data:` URIs and asset URIs such as `asset://ID` through unchanged and inline local files as base64. Ark accepts reference videos only as public URLs or `asset://` IDs.
|
|
154
|
+
|
|
155
|
+
## Development
|
|
156
|
+
|
|
157
|
+
```sh
|
|
158
|
+
uv sync --locked
|
|
159
|
+
uvx ruff check src/multimodal_mcp tests
|
|
160
|
+
uv run pytest
|
|
161
|
+
uv build
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
CI runs the tests on Linux and Windows. Changes to how omni agents run shell commands should also pass `uv run pytest tests/test_workspace.py` on Windows.
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# multimodal-mcp
|
|
2
|
+
|
|
3
|
+
One MCP server for image generation and editing, speech recognition, speech synthesis and voice design, video generation, and agents that analyze images, audio and video. Declare the models and their capabilities in a config file; the server registers a tool only when a configured model supports it, and picks the model for each call from what the call needs.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
The server requires Python 3.12 or later and runs over stdio:
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
uvx multimodal-mcp
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
or install it with `pip install multimodal-mcp` and run `multimodal-mcp`. For the development version, run `uvx --from git+https://github.com/Vigilans/multimodal-mcp multimodal-mcp`.
|
|
14
|
+
|
|
15
|
+
Register it with your MCP client as a stdio server named `multimodal`, for example:
|
|
16
|
+
|
|
17
|
+
```json
|
|
18
|
+
{
|
|
19
|
+
"mcpServers": {
|
|
20
|
+
"multimodal": {
|
|
21
|
+
"command": "uvx",
|
|
22
|
+
"args": ["multimodal-mcp", "--config", "/home/me/.agents/mcp/multimodal/config.toml"]
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The skill in [skills/multimodal](skills/multimodal/SKILL.md) tells agents how to choose models, write prompts, check results and deliver them. Install it with `npx skills add Vigilans/multimodal-mcp -s multimodal`, adding `-g` to install it globally.
|
|
29
|
+
|
|
30
|
+
On Windows, omni agents run their commands in Git Bash, so they require Git for Windows with `git` on `PATH`.
|
|
31
|
+
|
|
32
|
+
## Configuration
|
|
33
|
+
|
|
34
|
+
The config file is read from `--config <path>`, then `MULTIMODAL_MCP_CONFIG`, then `~/.agents/mcp/multimodal/config.toml`.
|
|
35
|
+
|
|
36
|
+
```toml
|
|
37
|
+
[providers.gateway]
|
|
38
|
+
endpoint = "openai"
|
|
39
|
+
base_url = "https://gateway.example.com/v1"
|
|
40
|
+
env_key = "GATEWAY_API_KEY" # or: api_key = "..."
|
|
41
|
+
|
|
42
|
+
[providers.gemini]
|
|
43
|
+
endpoint = "gemini" # base_url omitted: the official API
|
|
44
|
+
env_key = "GEMINI_API_KEY"
|
|
45
|
+
|
|
46
|
+
[[models.image]] # models.<image|asr|tts|video|omni>
|
|
47
|
+
provider = "gateway"
|
|
48
|
+
model = "gpt-image-2.5-flare"
|
|
49
|
+
description = "Fast image generation and editing"
|
|
50
|
+
capabilities = ["generate", "edit", "reference", "mask"]
|
|
51
|
+
|
|
52
|
+
[[models.video]]
|
|
53
|
+
provider = "gateway"
|
|
54
|
+
model = "MiniMax-H3"
|
|
55
|
+
description = "Video with synchronized audio; text, first/last frame, or reference driven"
|
|
56
|
+
capabilities = ["text", "frames"]
|
|
57
|
+
|
|
58
|
+
[[models.tts]] # capabilities from the API notes
|
|
59
|
+
provider = "gemini"
|
|
60
|
+
model = "gemini-3.8-flash-tts"
|
|
61
|
+
|
|
62
|
+
[[models.omni]] # omni models use an openai endpoint
|
|
63
|
+
provider = "gateway"
|
|
64
|
+
model = "gemini-3.5-flash"
|
|
65
|
+
capabilities = ["image", "audio"]
|
|
66
|
+
|
|
67
|
+
[defaults] # optional, per category
|
|
68
|
+
image = "gpt-image-2.5-flare"
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
A model without `capabilities` takes the list from [api-notes.toml](src/multimodal_mcp/catalog/api-notes.toml) for its endpoint and model ID, and the config fails to load when the API notes have none; the server treats unlisted capabilities as unsupported. Capabilities in the config must match what the endpoint implements for that model, since the server sends every call the listed capabilities allow. Tools, `list_models` and `defaults` name a model by its `model` value, or by `<provider>/<model>` when several entries share that value, such as `gateway/gpt-image-2.5-flare` beside `openai/gpt-image-2.5-flare`.
|
|
72
|
+
|
|
73
|
+
### Capabilities
|
|
74
|
+
|
|
75
|
+
| Category | Capability | Meaning |
|
|
76
|
+
|---|---|---|
|
|
77
|
+
| `image` | `generate`, `edit` | Create an image from a prompt; modify given images |
|
|
78
|
+
| | `reference` | Generate a new image following a reference's subject or style |
|
|
79
|
+
| | `mask` | Edit only a masked region |
|
|
80
|
+
| `asr` | `text` | Plain transcription |
|
|
81
|
+
| | `segment`, `word` | Timestamp granularity |
|
|
82
|
+
| | `speaker` | Speaker labels |
|
|
83
|
+
| | `hotwords` | A dedicated hotword field: vLLM `hotwords`, Gemini `custom_vocabulary` |
|
|
84
|
+
| `tts` | `voice` | Speak with a preset or registered voice ID |
|
|
85
|
+
| | `reference` | Imitate a voice sample sent with the request |
|
|
86
|
+
| | `timestamps` | Return segment and word timing of the generated speech |
|
|
87
|
+
| | `design` | Create a reusable voice ID from a description |
|
|
88
|
+
| `video` | `text` | Text-only generation |
|
|
89
|
+
| | `frames` | First frame, or first and last frames |
|
|
90
|
+
| | `reference` | Reference images, videos or audio |
|
|
91
|
+
| `omni` | `image`, `audio` | The media the model reads |
|
|
92
|
+
|
|
93
|
+
## Tools and results
|
|
94
|
+
|
|
95
|
+
| Tool | Registered when |
|
|
96
|
+
|---|---|
|
|
97
|
+
| `list_models(category?, capability?)` | always |
|
|
98
|
+
| `get_model_guide(model, lang, update?, api_notes?, prompt_guide?)` | always |
|
|
99
|
+
| `generate_image` | an `image` model has `generate` |
|
|
100
|
+
| `edit_image` | an `image` model has `edit` |
|
|
101
|
+
| `transcribe_audio` | an `asr` model has `text` |
|
|
102
|
+
| `transcribe_audio_with_timestamps` | an `asr` model has `segment` or `word` |
|
|
103
|
+
| `generate_speech` | a `tts` model has `voice` or `reference` |
|
|
104
|
+
| `voice_design` | a `tts` model has `design` |
|
|
105
|
+
| `generate_video`, `get_video_job` | a `video` model exists; both list every video model |
|
|
106
|
+
| `start_omni_agent`, `continue_omni_agent`, `get_omni_agent`, `wait_omni_agents`, `stop_omni_agent` | an `omni` model exists |
|
|
107
|
+
|
|
108
|
+
`get_model_guide` returns a model's API notes from [api-notes.toml](src/multimodal_mcp/catalog/api-notes.toml) and its family's prompt guide from [prompt-guides.toml](src/multimodal_mcp/catalog/prompt-guides.toml). The vendor reference files a guide names are downloaded into `~/.cache/multimodal-mcp/references/<family>/` on first use. Existing files are reused for either `lang`; `update=true` downloads them again in the requested `lang`, and a failed download keeps the existing file and returns an error. A deleted file is downloaded again on the next call.
|
|
109
|
+
|
|
110
|
+
Each tool's `model` parameter is an enum of the models that can perform it. Call arguments add requirements: `references` needs `reference`, `mask` needs `mask`, `granularity: "word"` needs `word`, `speakers` needs `speaker`, `hotwords` needs `hotwords`, `voice` and `reference_audio` need `voice` and `reference`, `timestamps` needs `timestamps`; a video with neither frames nor references needs `text`, `first_frame` and `last_frame` need `frames`, `reference_images`, `reference_videos` and `reference_audios` need `reference`. A named model that lacks a requirement is rejected before any request. Without `model`, the server uses the only capable model, else the category default if capable, else asks for a choice. Results report the model used.
|
|
111
|
+
|
|
112
|
+
`generate_image`, `edit_image`, `transcribe_audio`, `transcribe_audio_with_timestamps`, `generate_speech` and `generate_video` take `options`, merged into the endpoint's request body, for fields beyond the common parameters; the API notes list the useful ones.
|
|
113
|
+
|
|
114
|
+
Results are written to `~/.cache/multimodal-mcp/<YYYY-MM-DD>/<category>/` unless an absolute `output_path` is given, and tools return the file path. `generate_video` submits a job; `get_video_job` saves the video when the job completes, to its own `output_path` when given. Timestamped transcription and `generate_speech(timestamps=true)` write `{text, duration?, segments: [{start, end, text, speaker?}], words: [{start, end, word, speaker?}]}` with times in seconds, where either list may be empty.
|
|
115
|
+
|
|
116
|
+
Omni agents work in the background: `start_omni_agent` returns an `agent_id` at once, `wait_omni_agents` waits until an agent's run ends, and `get_omni_agent` returns its answer and a page of the tool calls it made. `continue_omni_agent` adds a message to the active run, or starts a new run after the previous one ended; `stop_omni_agent` cancels the current run. Agents live in server memory and are lost when the server restarts, and a provider request already in flight may complete and be billed after cancellation. They read files and run shell commands such as `ffmpeg` and `ffprobe`, taken from `PATH` or, when `PATH` lacks either, downloaded once with static-ffmpeg.
|
|
117
|
+
|
|
118
|
+
## Endpoints
|
|
119
|
+
|
|
120
|
+
`base_url` is the API root to which the endpoint appends its paths. When omitted, the endpoint's official API is used.
|
|
121
|
+
|
|
122
|
+
| Endpoint | Default `base_url` | Image | ASR | TTS | Video |
|
|
123
|
+
|---|---|---|---|---|---|
|
|
124
|
+
| `openai` | `https://api.openai.com/v1` | `/images/generations`, `/images/edits` | `/audio/transcriptions` | `/audio/speech` | `/videos`, multipart, as served by vLLM-Omni |
|
|
125
|
+
| `minimax` | `https://api.minimaxi.com` | `/v1/image_generation` (references are portrait subjects) | `/v1/speech_to_text` | `/v1/t2a_v2`, `/v1/voice_design` | `/v2/video_generation` |
|
|
126
|
+
| `ark` | `https://ark.cn-beijing.volces.com/api/v3` for image and video, `https://openspeech.bytedance.com/api/v3` for speech | `/images/generations` | `/sauc/bigmodel_nostream` WebSocket; needs `ffmpeg` on `PATH` | `/tts/unidirectional` | `/contents/generations/tasks` |
|
|
127
|
+
| `gemini` | `https://generativelanguage.googleapis.com/v1beta` | `/interactions` | `/interactions` | `/interactions`, `/voices` | `/interactions`, background |
|
|
128
|
+
|
|
129
|
+
Omni models use the `openai` endpoint's chat completions.
|
|
130
|
+
|
|
131
|
+
An `ark` `base_url` replaces both default roots (media and speech), so a provider with separate image/video and speech roots needs one provider entry per root. Include any plan prefix, such as `.../api/plan/v3` for Agent Plan images and video and `.../api/v3/plan` for Agent Plan speech. Ark speech models select their service with `X-Api-Resource-Id`: `doubao-seed-tts-2.0` and `doubao-seed-asr-2.0` map to `seed-tts-2.0` and `volc.seedasr.sauc.duration`; any other model name is sent as the resource ID.
|
|
132
|
+
|
|
133
|
+
`minimax` and `ark` video inputs pass URLs, `data:` URIs and asset URIs such as `asset://ID` through unchanged and inline local files as base64. Ark accepts reference videos only as public URLs or `asset://` IDs.
|
|
134
|
+
|
|
135
|
+
## Development
|
|
136
|
+
|
|
137
|
+
```sh
|
|
138
|
+
uv sync --locked
|
|
139
|
+
uvx ruff check src/multimodal_mcp tests
|
|
140
|
+
uv run pytest
|
|
141
|
+
uv build
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
CI runs the tests on Linux and Windows. Changes to how omni agents run shell commands should also pass `uv run pytest tests/test_workspace.py` on Windows.
|
|
145
|
+
|
|
146
|
+
## License
|
|
147
|
+
|
|
148
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "multimodal-mcp"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "MCP server for image, audio and video generation, transcription, voice design and media analysis."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.12"
|
|
9
|
+
dependencies = [
|
|
10
|
+
"fastmcp>=4.1.0",
|
|
11
|
+
"httpx>=0.28.1",
|
|
12
|
+
"openai>=3.28.0",
|
|
13
|
+
"pydantic>=2.14.0",
|
|
14
|
+
"pydantic-ai-harness>=0.66",
|
|
15
|
+
"pydantic-ai-slim[openai]>=2.66",
|
|
16
|
+
"static-ffmpeg>=3.0",
|
|
17
|
+
"websockets>=15",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[[project.authors]]
|
|
21
|
+
name = "Vigilans"
|
|
22
|
+
email = "vigilans@foxmail.com"
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Repository = "https://github.com/Vigilans/multimodal-mcp"
|
|
26
|
+
|
|
27
|
+
[project.scripts]
|
|
28
|
+
multimodal-mcp = "multimodal_mcp:main"
|
|
29
|
+
|
|
30
|
+
[build-system]
|
|
31
|
+
requires = ["uv_build>=0.12.13,<0.13.0"]
|
|
32
|
+
build-backend = "uv_build"
|
|
33
|
+
|
|
34
|
+
[dependency-groups]
|
|
35
|
+
dev = ["pytest>=9.2.0"]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "multimodal-mcp"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "MCP server for image, audio and video generation, transcription, voice design and media analysis."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Vigilans", email = "vigilans@foxmail.com" }
|
|
10
|
+
]
|
|
11
|
+
requires-python = ">=3.12"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"fastmcp>=4.1.0",
|
|
14
|
+
"httpx>=0.28.1",
|
|
15
|
+
"openai>=3.28.0",
|
|
16
|
+
"pydantic>=2.14.0",
|
|
17
|
+
"pydantic-ai-harness>=0.66",
|
|
18
|
+
"pydantic-ai-slim[openai]>=2.66",
|
|
19
|
+
"static-ffmpeg>=3.0",
|
|
20
|
+
"websockets>=15",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Repository = "https://github.com/Vigilans/multimodal-mcp"
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
multimodal-mcp = "multimodal_mcp:main"
|
|
28
|
+
|
|
29
|
+
[build-system]
|
|
30
|
+
requires = ["uv_build>=0.12.13,<0.13.0"]
|
|
31
|
+
build-backend = "uv_build"
|
|
32
|
+
|
|
33
|
+
[dependency-groups]
|
|
34
|
+
dev = [
|
|
35
|
+
"pytest>=9.2.0",
|
|
36
|
+
]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
|
|
3
|
+
from .config import load
|
|
4
|
+
from .server import build
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def main() -> None:
|
|
8
|
+
parser = argparse.ArgumentParser(prog="multimodal-mcp")
|
|
9
|
+
parser.add_argument("--config", help="Config file (default: ~/.agents/mcp/multimodal/config.toml)")
|
|
10
|
+
args = parser.parse_args()
|
|
11
|
+
build(load(args.config)).run(show_banner=False)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Model data looked up by endpoint and model ID: API notes and default capabilities in api-notes.toml, prompt guides
|
|
2
|
+
in prompt-guides.toml. Each file's header states its scope; data of another kind goes in a file of its own."""
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
# A trailing date version: -YYMMDD on Ark, -YYYY-MM-DD on OpenAI snapshots.
|
|
7
|
+
DATE = re.compile(r"-(\d{6}|\d{4}-\d{2}-\d{2})$")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def notes_name(model: str) -> str:
|
|
11
|
+
"""Catalog names compare case-insensitively, after the last "/" of a repo ID or path."""
|
|
12
|
+
return model.rsplit("/", 1)[-1].lower()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def matching_names(model: str) -> set[str]:
|
|
16
|
+
"""A model ID's names to match: with and without a trailing date version."""
|
|
17
|
+
name = notes_name(model)
|
|
18
|
+
return {name, DATE.sub("", name)}
|