video-context-mcp-server 0.48.2-beta → 0.49.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -32
- package/dist/generated/version.d.ts +1 -1
- package/dist/generated/version.js +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/services/mimoClient.d.ts +107 -0
- package/dist/services/mimoClient.d.ts.map +1 -0
- package/dist/services/mimoClient.js +251 -0
- package/dist/services/mimoClient.js.map +1 -0
- package/dist/services/providerRouter.d.ts +18 -14
- package/dist/services/providerRouter.d.ts.map +1 -1
- package/dist/services/providerRouter.js +28 -15
- package/dist/services/providerRouter.js.map +1 -1
- package/dist/tools/analyzeVideo.d.ts.map +1 -1
- package/dist/tools/analyzeVideo.js +86 -43
- package/dist/tools/analyzeVideo.js.map +1 -1
- package/dist/tools/orchestrator.d.ts +2 -0
- package/dist/tools/orchestrator.d.ts.map +1 -1
- package/dist/tools/orchestrator.js +12 -8
- package/dist/tools/orchestrator.js.map +1 -1
- package/dist/tools/schemas.d.ts +10 -10
- package/dist/tools/schemas.d.ts.map +1 -1
- package/dist/tools/schemas.js +10 -4
- package/dist/tools/schemas.js.map +1 -1
- package/dist/tools/searchTimestamp.d.ts.map +1 -1
- package/dist/tools/searchTimestamp.js +7 -1
- package/dist/tools/searchTimestamp.js.map +1 -1
- package/dist/tools/summarizeVideo.d.ts.map +1 -1
- package/dist/tools/summarizeVideo.js +45 -9
- package/dist/tools/summarizeVideo.js.map +1 -1
- package/dist/utils/videoUtils.d.ts.map +1 -1
- package/dist/utils/videoUtils.js +8 -1
- package/dist/utils/videoUtils.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ An MCP server that gives coding assistants (GitHub Copilot, Cursor, Claude Code)
|
|
|
15
15
|
- 🎙️ **Audio Transcription** — Transcribe speech with paragraph-level timestamps (`[MM:SS]`) or export as SRT/VTT subtitles and JSON using Deepgram, AssemblyAI, Groq/Whisper, or Gemini
|
|
16
16
|
- 🔊 **Speaker Diarization** — Identify who said what (Deepgram and AssemblyAI)
|
|
17
17
|
- 🔊 **Audio-Enhanced Analysis** — Auto-transcribes audio and injects transcripts into AI prompts for richer results (GLM/Kimi/Qwen)
|
|
18
|
-
- 🔄 **Multi-Provider Support** — Choose between GLM-4.6V, Qwen3.5, Kimi K2.5, or
|
|
18
|
+
- 🔄 **Multi-Provider Support** — Choose between GLM-4.6V, Qwen3.5, Kimi K2.5, Gemini, or MiMo-V2 Omni
|
|
19
19
|
- 🎯 **Smart Video Handling** — Extracts keyframes from long videos to reduce token usage
|
|
20
20
|
- ⭐ **Pro tier** — Extended frame extraction, multi-platform downloads, higher resolution. [Learn more ↓](#pro)
|
|
21
21
|
|
|
@@ -65,6 +65,7 @@ Create or update `.vscode/mcp.json` in your workspace:
|
|
|
65
65
|
"DASHSCOPE_API_KEY": "your-dashscope-key",
|
|
66
66
|
"MOONSHOT_API_KEY": "your-moonshot-key",
|
|
67
67
|
"GEMINI_API_KEY": "your-gemini-key",
|
|
68
|
+
"MIMO_API_KEY": "your-mimo-key",
|
|
68
69
|
"DEEPGRAM_API_KEY": "your-deepgram-key",
|
|
69
70
|
"ASSEMBLYAI_API_KEY": "your-assemblyai-key",
|
|
70
71
|
"GROQ_API_KEY": "your-groq-key"
|
|
@@ -93,6 +94,7 @@ Add to your Cursor MCP configuration (global or project-level):
|
|
|
93
94
|
"DASHSCOPE_API_KEY": "your-dashscope-key",
|
|
94
95
|
"MOONSHOT_API_KEY": "your-moonshot-key",
|
|
95
96
|
"GEMINI_API_KEY": "your-gemini-key",
|
|
97
|
+
"MIMO_API_KEY": "your-mimo-key",
|
|
96
98
|
"DEEPGRAM_API_KEY": "your-deepgram-key",
|
|
97
99
|
"ASSEMBLYAI_API_KEY": "your-assemblyai-key",
|
|
98
100
|
"GROQ_API_KEY": "your-groq-key"
|
|
@@ -115,6 +117,7 @@ claude mcp add \
|
|
|
115
117
|
--env DASHSCOPE_API_KEY=your-dashscope-key \
|
|
116
118
|
--env MOONSHOT_API_KEY=your-moonshot-key \
|
|
117
119
|
--env GEMINI_API_KEY=your-gemini-key \
|
|
120
|
+
--env MIMO_API_KEY=your-mimo-key \
|
|
118
121
|
--env DEEPGRAM_API_KEY=your-deepgram-key \
|
|
119
122
|
--env ASSEMBLYAI_API_KEY=your-assemblyai-key \
|
|
120
123
|
--env GROQ_API_KEY=your-groq-key \
|
|
@@ -139,6 +142,7 @@ Create `.mcp.json` in your project root:
|
|
|
139
142
|
"DASHSCOPE_API_KEY": "your-dashscope-key",
|
|
140
143
|
"MOONSHOT_API_KEY": "your-moonshot-key",
|
|
141
144
|
"GEMINI_API_KEY": "your-gemini-key",
|
|
145
|
+
"MIMO_API_KEY": "your-mimo-key",
|
|
142
146
|
"DEEPGRAM_API_KEY": "your-deepgram-key",
|
|
143
147
|
"ASSEMBLYAI_API_KEY": "your-assemblyai-key",
|
|
144
148
|
"GROQ_API_KEY": "your-groq-key"
|
|
@@ -169,6 +173,7 @@ Use `npx -y video-context-mcp-server@latest` as the command instead of `video-co
|
|
|
169
173
|
"DASHSCOPE_API_KEY": "your-dashscope-key",
|
|
170
174
|
"MOONSHOT_API_KEY": "your-moonshot-key",
|
|
171
175
|
"GEMINI_API_KEY": "your-gemini-key",
|
|
176
|
+
"MIMO_API_KEY": "your-mimo-key",
|
|
172
177
|
"DEEPGRAM_API_KEY": "your-deepgram-key",
|
|
173
178
|
"ASSEMBLYAI_API_KEY": "your-assemblyai-key",
|
|
174
179
|
"GROQ_API_KEY": "your-groq-key"
|
|
@@ -186,6 +191,7 @@ claude mcp add \
|
|
|
186
191
|
--env DASHSCOPE_API_KEY=your-dashscope-key \
|
|
187
192
|
--env MOONSHOT_API_KEY=your-moonshot-key \
|
|
188
193
|
--env GEMINI_API_KEY=your-gemini-key \
|
|
194
|
+
--env MIMO_API_KEY=your-mimo-key \
|
|
189
195
|
--env DEEPGRAM_API_KEY=your-deepgram-key \
|
|
190
196
|
--env ASSEMBLYAI_API_KEY=your-assemblyai-key \
|
|
191
197
|
--env GROQ_API_KEY=your-groq-key \
|
|
@@ -253,7 +259,7 @@ All tools accept local file paths, `file://` URIs, and remote `http(s)` URLs. Re
|
|
|
253
259
|
|
|
254
260
|
### Video Providers
|
|
255
261
|
|
|
256
|
-
Set all keys to get the full fallback chain. The server will try GLM first, then Gemini, then Qwen, then Kimi, so having all keys ensures it never gets stuck without a working provider:
|
|
262
|
+
Set all keys to get the full fallback chain. The server will try GLM first, then Gemini, then Qwen, then Kimi, then MiMo, so having all keys ensures it never gets stuck without a working provider:
|
|
257
263
|
|
|
258
264
|
| Provider | Key | Link |
|
|
259
265
|
| -------------------------------------- | ------------------- | --------------------------------------------------------------------------------------------- |
|
|
@@ -261,13 +267,15 @@ Set all keys to get the full fallback chain. The server will try GLM first, then
|
|
|
261
267
|
| **Gemini 3 Flash Preview** (free-tier) | `GEMINI_API_KEY` | [Get key](https://aistudio.google.com/app/apikey) |
|
|
262
268
|
| **Qwen3.5** (free credits) | `DASHSCOPE_API_KEY` | [Get key](https://modelstudio.console.alibabacloud.com/ap-southeast-1?tab=dashboard#/api-key) |
|
|
263
269
|
| **Kimi K2.5** | `MOONSHOT_API_KEY` | [Get key](https://platform.moonshot.ai) |
|
|
270
|
+
| **MiMo-V2 Omni** (free launch promo) | `MIMO_API_KEY` | [Get key](https://platform.xiaomimimo.com/#/console/api-keys) |
|
|
264
271
|
|
|
265
272
|
When a provider's API key is missing or its API call fails at runtime, tools automatically fall back through the remaining providers in priority order, starting from the configured default:
|
|
266
273
|
|
|
267
|
-
- GLM default (standard): **GLM → Gemini → Qwen → Kimi**
|
|
268
|
-
- Gemini default: **Gemini → GLM → Qwen → Kimi**
|
|
269
|
-
- Qwen default: **Qwen → GLM → Gemini → Kimi**
|
|
270
|
-
- Kimi default: **Kimi → GLM → Gemini → Qwen**
|
|
274
|
+
- GLM default (standard): **GLM → Gemini → Qwen → Kimi → MiMo**
|
|
275
|
+
- Gemini default: **Gemini → GLM → Qwen → Kimi → MiMo**
|
|
276
|
+
- Qwen default: **Qwen → GLM → Gemini → Kimi → MiMo**
|
|
277
|
+
- Kimi default: **Kimi → GLM → Gemini → Qwen → MiMo**
|
|
278
|
+
- MiMo default: **MiMo → GLM → Gemini → Qwen → Kimi**
|
|
271
279
|
|
|
272
280
|
### Audio Providers
|
|
273
281
|
|
|
@@ -288,17 +296,17 @@ When an audio key is missing or an audio API call fails at runtime, tools automa
|
|
|
288
296
|
|
|
289
297
|
### Video Providers
|
|
290
298
|
|
|
291
|
-
| Feature | GLM-4.6V (default) | Gemini 3 Flash Preview | Qwen3.5 | Kimi K2.5 |
|
|
292
|
-
| -------------- | ------------------------------------------------------ | ---------------------------------------------- | ------------------------------------------------------ | ---------------------------------------------- |
|
|
293
|
-
| Price | Free tier available (GLM-4.6V-Flash) | Free tier available | Free credits (1M tokens, 90 days) | $0.60 input / $3.00 output per 1M tokens |
|
|
294
|
-
| Video formats | mp4, avi, mov, wmv, webm, m4v | mp4, mpeg, mov, avi, flv, mpg, webm, wmv, 3gpp | mp4, avi, mov, wmv, webm, m4v | mp4, mpeg, mov, avi, flv, mpg, webm, wmv, 3gpp |
|
|
295
|
-
| Context window | 128K tokens | 1M tokens | 1M tokens | 256K tokens |
|
|
296
|
-
| Max file size | ~12 MB base64 / frames fallback / **unlimited w/ S3↓** | 2 GB | ~10 MB base64 / frames fallback / **unlimited w/ S3↓** | 100 MB |
|
|
297
|
-
| Best for | **Default** — free, no card required | Free fallback, no card (less accurate) | Free credits (card required) | Paid — broadest format support |
|
|
299
|
+
| Feature | GLM-4.6V (default) | Gemini 3 Flash Preview | Qwen3.5 | Kimi K2.5 | MiMo-V2 Omni |
|
|
300
|
+
| -------------- | ------------------------------------------------------ | ---------------------------------------------- | ------------------------------------------------------ | ---------------------------------------------- | ------------------------------------------------------ |
|
|
301
|
+
| Price | Free tier available (GLM-4.6V-Flash) | Free tier available | Free credits (1M tokens, 90 days) | $0.60 input / $3.00 output per 1M tokens | Free launch promo |
|
|
302
|
+
| Video formats | mp4, avi, mov, wmv, webm, m4v | mp4, mpeg, mov, avi, flv, mpg, webm, wmv, 3gpp | mp4, avi, mov, wmv, webm, m4v | mp4, mpeg, mov, avi, flv, mpg, webm, wmv, 3gpp | mp4, mov, avi, wmv |
|
|
303
|
+
| Context window | 128K tokens | 1M tokens | 1M tokens | 256K tokens | 256K tokens |
|
|
304
|
+
| Max file size | ~12 MB base64 / frames fallback / **unlimited w/ S3↓** | 2 GB | ~10 MB base64 / frames fallback / **unlimited w/ S3↓** | 100 MB | ~10 MB base64 / frames fallback / **unlimited w/ S3↓** |
|
|
305
|
+
| Best for | **Default** — free, no card required | Free fallback, no card (less accurate) | Free credits (card required) | Paid — broadest format support | Free launch promo; thinking mode |
|
|
298
306
|
|
|
299
|
-
**GLM-4.6V** is the default — it offers a free tier with no credit card required. **Gemini** is the second fallback — also free with no card required, though it has proven less accurate for video analysis in practice. **Qwen3.5** offers free credits (1M tokens for 90 days) but requires a credit card to sign up. **Kimi K2.5** is a paid provider with the broadest video format support.
|
|
307
|
+
**GLM-4.6V** is the default — it offers a free tier with no credit card required. **Gemini** is the second fallback — also free with no card required, though it has proven less accurate for video analysis in practice. **Qwen3.5** offers free credits (1M tokens for 90 days) but requires a credit card to sign up. **Kimi K2.5** is a paid provider with the broadest video format support. **MiMo-V2 Omni** is Xiaomi's multimodal model with thinking mode support and a free launch promotion.
|
|
300
308
|
|
|
301
|
-
Set `VIDEO_MCP_DEFAULT_PROVIDER=gemini`, `qwen`, or `
|
|
309
|
+
Set `VIDEO_MCP_DEFAULT_PROVIDER=gemini`, `qwen`, `kimi`, or `mimo` to change the default.
|
|
302
310
|
|
|
303
311
|
> **ℹ️ Large local files with GLM or Qwen:** Both providers have a 10–12 MB base64 limit for local files. When a file exceeds this limit, the server first tries to fall back to an upload-capable provider (Gemini or Kimi) if one is available in the fallback chain. Frame-based analysis (evenly-spaced keyframes sent as images) is used only as a **last resort** when no upload-capable provider is available, or when all upload-capable providers fail at runtime — no configuration needed. For the highest quality with large local videos, set up the optional **S3 relay** (below) — GLM and Qwen will receive a presigned URL to the full video, bypassing the limit entirely and taking priority over both fallbacks.
|
|
304
312
|
|
|
@@ -376,13 +384,14 @@ Set `AUDIO_MCP_DEFAULT_PROVIDER` to change the default.
|
|
|
376
384
|
|
|
377
385
|
### Core
|
|
378
386
|
|
|
379
|
-
| Variable | Description
|
|
380
|
-
| ---------------------------- |
|
|
381
|
-
| `Z_AI_API_KEY` | Z.AI API key for GLM-4.6V
|
|
382
|
-
| `DASHSCOPE_API_KEY` | Alibaba Cloud API key for Qwen3.5
|
|
383
|
-
| `MOONSHOT_API_KEY` | Moonshot AI API key for Kimi K2.5
|
|
384
|
-
| `GEMINI_API_KEY` | Google API key for Gemini
|
|
385
|
-
| `
|
|
387
|
+
| Variable | Description | Default |
|
|
388
|
+
| ---------------------------- | ------------------------------------------------------------------ | ------- |
|
|
389
|
+
| `Z_AI_API_KEY` | Z.AI API key for GLM-4.6V | — |
|
|
390
|
+
| `DASHSCOPE_API_KEY` | Alibaba Cloud API key for Qwen3.5 | — |
|
|
391
|
+
| `MOONSHOT_API_KEY` | Moonshot AI API key for Kimi K2.5 | — |
|
|
392
|
+
| `GEMINI_API_KEY` | Google API key for Gemini | — |
|
|
393
|
+
| `MIMO_API_KEY` | Xiaomi MiMo API key for MiMo-V2 Omni | — |
|
|
394
|
+
| `VIDEO_MCP_DEFAULT_PROVIDER` | Default video provider: `glm`, `qwen`, `kimi`, `gemini`, or `mimo` | `glm` |
|
|
386
395
|
|
|
387
396
|
### S3 Relay
|
|
388
397
|
|
|
@@ -396,13 +405,13 @@ Set `AUDIO_MCP_DEFAULT_PROVIDER` to change the default.
|
|
|
396
405
|
|
|
397
406
|
### Audio
|
|
398
407
|
|
|
399
|
-
| Variable | Description
|
|
400
|
-
| ------------------------------ |
|
|
401
|
-
| `DEEPGRAM_API_KEY` | Deepgram API key
|
|
402
|
-
| `ASSEMBLYAI_API_KEY` | AssemblyAI API key
|
|
403
|
-
| `GROQ_API_KEY` | Groq API key for Whisper transcription
|
|
404
|
-
| `AUDIO_MCP_DEFAULT_PROVIDER` | Default audio provider: `deepgram`, `assemblyai`, `groq`, or `gemini`. Falls back in that order when the selected key is missing.
|
|
405
|
-
| `AUDIO_ENHANCE_VIDEO_ANALYSIS` | Inject audio transcripts into `analyze_video`/`summarize_video` prompts (GLM/Kimi/Qwen). `auto` = transcribe when audio track detected; `true` = always; `false` = disabled. | `auto` |
|
|
408
|
+
| Variable | Description | Default |
|
|
409
|
+
| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
|
|
410
|
+
| `DEEPGRAM_API_KEY` | Deepgram API key | — |
|
|
411
|
+
| `ASSEMBLYAI_API_KEY` | AssemblyAI API key | — |
|
|
412
|
+
| `GROQ_API_KEY` | Groq API key for Whisper transcription | — |
|
|
413
|
+
| `AUDIO_MCP_DEFAULT_PROVIDER` | Default audio provider: `deepgram`, `assemblyai`, `groq`, or `gemini`. Falls back in that order when the selected key is missing. | `deepgram` |
|
|
414
|
+
| `AUDIO_ENHANCE_VIDEO_ANALYSIS` | Inject audio transcripts into `analyze_video`/`summarize_video` prompts (GLM/Kimi/Qwen/MiMo). `auto` = transcribe when audio track detected; `true` = always; `false` = disabled. | `auto` |
|
|
406
415
|
|
|
407
416
|
### Caching
|
|
408
417
|
|
|
@@ -477,9 +486,9 @@ vmcp cache clear:all --yes # skip confirmation
|
|
|
477
486
|
|
|
478
487
|
### Video Summarization
|
|
479
488
|
|
|
480
|
-
| Variable | Description
|
|
481
|
-
| ---------------------- |
|
|
482
|
-
| `VIDEO_MCP_MAX_FRAMES` | Max keyframes for `summarize_video` (GLM/Kimi/Qwen only; Gemini uploads full video). **Free**: clamped 5–50. **Pro**: default 100; set `0` for uncapped. Either way, trailing frames are automatically dropped if the total payload exceeds the provider's size limit (12
|
|
489
|
+
| Variable | Description | Default |
|
|
490
|
+
| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
|
|
491
|
+
| `VIDEO_MCP_MAX_FRAMES` | Max keyframes for `summarize_video` (GLM/Kimi/Qwen/MiMo only; Gemini uploads full video). **Free**: clamped 5–50. **Pro**: default 100; set `0` for uncapped. Either way, trailing frames are automatically dropped if the total payload exceeds the provider's size limit (12 MB for GLM, 10 MB for Qwen/MiMo, 80 MB for Kimi). | `50` free / `100` pro |
|
|
483
492
|
|
|
484
493
|
### Qwen
|
|
485
494
|
|
|
@@ -494,6 +503,12 @@ vmcp cache clear:all --yes # skip confirmation
|
|
|
494
503
|
| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- |
|
|
495
504
|
| `GLM_REQUEST_TIMEOUT_MS` | Client-side timeout (ms) for GLM API requests. Z.AI has no hard server-side streaming timeout, so raising this value (e.g. `480000` for 8 minutes) can help when analysing long videos with many frames. | `240000` (4 min) |
|
|
496
505
|
|
|
506
|
+
### MiMo
|
|
507
|
+
|
|
508
|
+
| Variable | Description | Default |
|
|
509
|
+
| ------------------------- | ------------------------------------------------------------------------------ | ---------------- |
|
|
510
|
+
| `MIMO_REQUEST_TIMEOUT_MS` | Client-side timeout (ms) for MiMo API requests. Default: `240000` (4 minutes). | `240000` (4 min) |
|
|
511
|
+
|
|
497
512
|
### yt-dlp (Platform Downloads)
|
|
498
513
|
|
|
499
514
|
| Variable | Description | Default |
|
|
@@ -524,6 +539,7 @@ vmcp cache clear:all --yes # skip confirmation
|
|
|
524
539
|
"DASHSCOPE_API_KEY": "your-dashscope-key",
|
|
525
540
|
"MOONSHOT_API_KEY": "your-moonshot-key",
|
|
526
541
|
"GEMINI_API_KEY": "your-gemini-key",
|
|
542
|
+
"MIMO_API_KEY": "your-mimo-key",
|
|
527
543
|
"DEEPGRAM_API_KEY": "your-deepgram-key",
|
|
528
544
|
"ASSEMBLYAI_API_KEY": "your-assemblyai-key",
|
|
529
545
|
"GROQ_API_KEY": "your-groq-key",
|
|
@@ -662,6 +678,7 @@ Use this `.vscode/mcp.json` to run the local build (never commit this file):
|
|
|
662
678
|
"DASHSCOPE_API_KEY": "your-dashscope-key",
|
|
663
679
|
"MOONSHOT_API_KEY": "your-moonshot-key",
|
|
664
680
|
"GEMINI_API_KEY": "your-gemini-key",
|
|
681
|
+
"MIMO_API_KEY": "your-mimo-key",
|
|
665
682
|
"VIDEO_MCP_DEFAULT_PROVIDER": "glm",
|
|
666
683
|
"DEEPGRAM_API_KEY": "your-deepgram-key",
|
|
667
684
|
"ASSEMBLYAI_API_KEY": "your-assemblyai-key",
|
|
@@ -708,6 +725,7 @@ Proprietary — All Rights Reserved. No part of this software may be copied, mod
|
|
|
708
725
|
- [Kimi K2.5](https://github.com/MoonshotAI/Kimi-K2.5) by Moonshot AI
|
|
709
726
|
- [GLM-4.6V](https://docs.z.ai/guides/vlm/glm-4.6v) by Z.AI
|
|
710
727
|
- [Qwen3.5](https://bailian.console.alibabacloud.com/ap-southeast-1/) by Alibaba Cloud
|
|
728
|
+
- [MiMo-V2 Omni](https://platform.xiaomimimo.com/) by Xiaomi
|
|
711
729
|
- [Deepgram](https://www.deepgram.com/) for audio transcription
|
|
712
730
|
- [AssemblyAI](https://www.assemblyai.com/) for audio transcription
|
|
713
731
|
- [Groq/Whisper](https://console.groq.com/) for audio transcription
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const VERSION = "0.
|
|
1
|
+
export declare const VERSION = "0.49.0-beta";
|
|
2
2
|
//# sourceMappingURL=version.d.ts.map
|
package/dist/index.js
CHANGED
|
@@ -63,7 +63,7 @@ async function main() {
|
|
|
63
63
|
// Tool 2: summarize_video - Generate structured video summary
|
|
64
64
|
server.registerTool('summarize_video', {
|
|
65
65
|
title: 'Summarize Video',
|
|
66
|
-
description: "Generate a structured summary of the video including overview, key scenes, and timeline. For long videos (>5 min), extracts keyframes to reduce token usage (unless using Gemini, which processes natively). Provider options: 'glm' (default), 'qwen', 'kimi', 'gemini'.",
|
|
66
|
+
description: "Generate a structured summary of the video including overview, key scenes, and timeline. For long videos (>5 min), extracts keyframes to reduce token usage (unless using Gemini, which processes natively). Provider options: 'glm' (default), 'qwen', 'kimi', 'gemini', 'mimo'.",
|
|
67
67
|
inputSchema: summarizeVideoSchema,
|
|
68
68
|
}, withUpdateCheck(summarizeVideoTool));
|
|
69
69
|
// Tool 3: extract_frames - Extract frames from video
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAEhF,uBAAuB;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAA;AAC5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAA;AAChE,OAAO,EACL,sBAAsB,EACtB,mBAAmB,GACpB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAA;AAEhD,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,oBAAoB,CAAA;AAE3B;;;GAGG;AAEH,KAAK,UAAU,IAAI;IACjB,0CAA0C;IAC1C,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,OAAO;KACjB,EACD;QACE,YAAY,EAAE;YACZ,OAAO,EAAE,EAAE,EAAE,wCAAwC;SACtD;KACF,CACF,CAAA;IAED,8DAA8D;IAC9D,eAAe,CAAC,MAAM,CAAC,CAAA;IAEvB,wGAAwG;IACxG,sBAAsB,EAAE,CAAA;IAExB,wGAAwG;IACxG,mBAAmB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;IAExD,kFAAkF;IAClF,kBAAkB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;IAEvD,0EAA0E;IAC1E,qBAAqB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;IAE1D,8EAA8E;IAC9E,MAAM,qBAAqB,EAAE,CAAA;IAE7B;;;;OAIG;IACH,MAAM,eAAe,GACnB,CAAI,OAA+C,EAAE,EAAE,CACvD,KAAK,EAAE,MAAS,EAA2B,EAAE;QAC3C,kBAAkB,EAAE,CAAA;QACpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAA;QACpC,MAAM,kBAAkB,EAAE,CAAA;QAC1B,OAAO,oBAAoB,CAAC,MAAM,CAAC,CAAA;IACrC,CAAC,CAAA;IAEH,oCAAoC;IAEpC,4DAA4D;IAC5D,MAAM,CAAC,YAAY,CACjB,eAAe,EACf;QACE,KAAK,EAAE,eAAe;QACtB,WAAW,EACT,mGAAmG;QACrG,WAAW,EAAE,kBAAkB;KAChC,EACD,eAAe,CAAC,gBAAgB,CAAC,CAClC,CAAA;IAED,8DAA8D;IAC9D,MAAM,CAAC,YAAY,CACjB,iBAAiB,EACjB;QACE,KAAK,EAAE,iBAAiB;QACxB,WAAW,EACT,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAEhF,uBAAuB;AACvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAA;AAC5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAA;AAChE,OAAO,EACL,sBAAsB,EACtB,mBAAmB,GACpB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAA;AAEhD,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,oBAAoB,CAAA;AAE3B;;;GAGG;AAEH,KAAK,UAAU,IAAI;IACjB,0CAA0C;IAC1C,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,OAAO;KACjB,EACD;QACE,YAAY,EAAE;YACZ,OAAO,EAAE,EAAE,EAAE,wCAAwC;SACtD;KACF,CACF,CAAA;IAED,8DAA8D;IAC9D,eAAe,CAAC,MAAM,CAAC,CAAA;IAEvB,wGAAwG;IACxG,sBAAsB,EAAE,CAAA;IAExB,wGAAwG;IACxG,mBAAmB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;IAExD,kFAAkF;IAClF,kBAAkB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;IAEvD,0EAA0E;IAC1E,qBAAqB,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;IAE1D,8EAA8E;IAC9E,MAAM,qBAAqB,EAAE,CAAA;IAE7B;;;;OAIG;IACH,MAAM,eAAe,GACnB,CAAI,OAA+C,EAAE,EAAE,CACvD,KAAK,EAAE,MAAS,EAA2B,EAAE;QAC3C,kBAAkB,EAAE,CAAA;QACpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAA;QACpC,MAAM,kBAAkB,EAAE,CAAA;QAC1B,OAAO,oBAAoB,CAAC,MAAM,CAAC,CAAA;IACrC,CAAC,CAAA;IAEH,oCAAoC;IAEpC,4DAA4D;IAC5D,MAAM,CAAC,YAAY,CACjB,eAAe,EACf;QACE,KAAK,EAAE,eAAe;QACtB,WAAW,EACT,mGAAmG;QACrG,WAAW,EAAE,kBAAkB;KAChC,EACD,eAAe,CAAC,gBAAgB,CAAC,CAClC,CAAA;IAED,8DAA8D;IAC9D,MAAM,CAAC,YAAY,CACjB,iBAAiB,EACjB;QACE,KAAK,EAAE,iBAAiB;QACxB,WAAW,EACT,mRAAmR;QACrR,WAAW,EAAE,oBAAoB;KAClC,EACD,eAAe,CAAC,kBAAkB,CAAC,CACpC,CAAA;IAED,qDAAqD;IACrD,MAAM,CAAC,YAAY,CACjB,gBAAgB,EAChB;QACE,KAAK,EAAE,gBAAgB;QACvB,WAAW,EACT,iKAAiK;QACnK,WAAW,EAAE,mBAAmB;KACjC,EACD,eAAe,CAAC,iBAAiB,CAAC,CACnC,CAAA;IAED,kEAAkE;IAClE,MAAM,CAAC,YAAY,CACjB,kBAAkB,EAClB;QACE,KAAK,EAAE,kBAAkB;QACzB,WAAW,EACT,0LAA0L;QAC5L,WAAW,EAAE,qBAAqB;KACnC,EACD,eAAe,CAAC,mBAAmB,CAAC,CACrC,CAAA;IAED,8CAA8C;IAC9C,MAAM,CAAC,YAAY,CACjB,gBAAgB,EAChB;QACE,KAAK,EAAE,gBAAgB;QACvB,WAAW,EACT,sLAAsL;QACxL,WAAW,EAAE,kBAAkB;KAChC,EACD,eAAe,CAAC,gBAAgB,CAAC,CAClC,CAAA;IAED,2DAA2D;IAC3D,MAAM,CAAC,YAAY,CACjB,kBAAkB,EAClB;QACE,KAAK,EAAE,kBAAkB;QACzB,WAAW,EACT,kMAAkM;QACpM,WAAW,EAAE,qBAAqB;KACnC,EACD,eAAe,CAAC,mBAAmB,CAAC,CACrC,CAAA;IAED,yCAAyC;IACzC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAA;IAC5C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IAE/B,+DAA+D;AACjE,CAAC;AAED,0EAA0E;AAC1E,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;IAChC,MAAM,CAAC,qBAAqB,CAAC;SAC1B,IAAI,CAAC,CAAC,EAAE,kBAAkB,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3E,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QACtB,OAAO,CAAC,KAAK,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;AACN,CAAC;KAAM,CAAC;IACN,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACrB,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MiMo-V2 Omni Client
|
|
3
|
+
* Handles video analysis using Xiaomi's MiMo API
|
|
4
|
+
*
|
|
5
|
+
* API Specification (Confirmed):
|
|
6
|
+
* - Base URL: https://api.xiaomimimo.com/v1
|
|
7
|
+
* - Endpoint: https://api.xiaomimimo.com/v1/chat/completions
|
|
8
|
+
* - Auth: api-key header or Authorization: Bearer
|
|
9
|
+
* - Models: mimo-v2-omni (video), mimo-v2-pro, mimo-v2-flash, mimo-v2-tts
|
|
10
|
+
* - Video: URL (max 300MB) or Base64 (max 10MB)
|
|
11
|
+
* - Thinking: enabled by default for omni/pro
|
|
12
|
+
*/
|
|
13
|
+
type MimoContentPart = {
|
|
14
|
+
type: 'text';
|
|
15
|
+
text: string;
|
|
16
|
+
} | {
|
|
17
|
+
type: 'video_url';
|
|
18
|
+
video_url: {
|
|
19
|
+
url: string;
|
|
20
|
+
};
|
|
21
|
+
fps?: number;
|
|
22
|
+
media_resolution?: string;
|
|
23
|
+
} | {
|
|
24
|
+
type: 'image_url';
|
|
25
|
+
image_url: {
|
|
26
|
+
url: string;
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
export interface MimoClientOptions {
|
|
30
|
+
apiKey: string;
|
|
31
|
+
baseURL?: string;
|
|
32
|
+
}
|
|
33
|
+
export declare class MimoClient {
|
|
34
|
+
private apiKey;
|
|
35
|
+
private baseURL;
|
|
36
|
+
constructor(options: MimoClientOptions);
|
|
37
|
+
/**
|
|
38
|
+
* Analyze a video using base64-encoded data
|
|
39
|
+
* @param base64Data - Base64-encoded video data
|
|
40
|
+
* @param prompt - The prompt/question to ask about the video
|
|
41
|
+
* @param mimeType - MIME type of the video (default: 'video/mp4')
|
|
42
|
+
* @param usePro - Use the pro model variant (default: false, uses standard mimo-v2-omni)
|
|
43
|
+
* @returns The AI analysis response
|
|
44
|
+
*/
|
|
45
|
+
analyzeVideoBase64(base64Data: string, prompt: string, mimeType?: string, usePro?: boolean): Promise<string>;
|
|
46
|
+
/**
|
|
47
|
+
* Analyze a video from a URL
|
|
48
|
+
* @param url - URL of the video
|
|
49
|
+
* @param prompt - The prompt/question to ask about the video
|
|
50
|
+
* @param fps - Server-side frame extraction rate (default: 2.0)
|
|
51
|
+
* @param usePro - Use the pro model variant (default: false, uses mimo-v2-omni)
|
|
52
|
+
* @returns The AI analysis response
|
|
53
|
+
*/
|
|
54
|
+
analyzeVideoUrl(url: string, prompt: string, fps?: number, usePro?: boolean): Promise<string>;
|
|
55
|
+
/**
|
|
56
|
+
* Analyze multiple images (e.g., extracted video frames)
|
|
57
|
+
* Used for the long-video frame-based strategy
|
|
58
|
+
* @param images - Array of image data with MIME types
|
|
59
|
+
* @param prompt - The prompt/question
|
|
60
|
+
* @returns The AI analysis response
|
|
61
|
+
*/
|
|
62
|
+
analyzeImages(images: Array<{
|
|
63
|
+
data: string;
|
|
64
|
+
mimeType: string;
|
|
65
|
+
}>, prompt: string): Promise<string>;
|
|
66
|
+
/**
|
|
67
|
+
* Analyze with thinking mode enabled
|
|
68
|
+
* Returns both the chain-of-thought reasoning and the final answer
|
|
69
|
+
* @param content - Content parts (video or images)
|
|
70
|
+
* @param prompt - The prompt/question
|
|
71
|
+
* @param usePro - Use the pro model variant
|
|
72
|
+
* @returns Object with reasoning and answer
|
|
73
|
+
*/
|
|
74
|
+
analyzeWithThinking(content: MimoContentPart[], prompt: string, usePro?: boolean): Promise<{
|
|
75
|
+
reasoning: string;
|
|
76
|
+
answer: string;
|
|
77
|
+
}>;
|
|
78
|
+
/**
|
|
79
|
+
* Make a chat completion request to the MiMo API
|
|
80
|
+
* @param options - Request options including model, messages, and thinking settings
|
|
81
|
+
* @returns Response object and parsed data
|
|
82
|
+
*/
|
|
83
|
+
private requestCompletion;
|
|
84
|
+
/**
|
|
85
|
+
* Make a chat completion request with standard error handling
|
|
86
|
+
* @param options - Request options
|
|
87
|
+
* @param usePro - Use pro model (for convenience methods)
|
|
88
|
+
* @returns The text response
|
|
89
|
+
*/
|
|
90
|
+
private requestChatCompletion;
|
|
91
|
+
/**
|
|
92
|
+
* Extract text from message content
|
|
93
|
+
* Handles both string content and array content formats
|
|
94
|
+
* @param content - The message content
|
|
95
|
+
* @returns Extracted text
|
|
96
|
+
*/
|
|
97
|
+
private extractTextFromContent;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Create a MiMo client if API key is available
|
|
101
|
+
* @param apiKey - MiMo API key (MIMO_API_KEY env var)
|
|
102
|
+
* @param baseURL - Optional custom base URL
|
|
103
|
+
* @returns MiMoClient instance or null if no API key
|
|
104
|
+
*/
|
|
105
|
+
export declare function createMimoClient(apiKey?: string, baseURL?: string): MimoClient | null;
|
|
106
|
+
export {};
|
|
107
|
+
//# sourceMappingURL=mimoClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mimoClient.d.ts","sourceRoot":"","sources":["../../src/services/mimoClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAiBH,KAAK,eAAe,GAChB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IACE,IAAI,EAAE,WAAW,CAAA;IACjB,SAAS,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B,GACD;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAA;AAoBrD,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,OAAO,CAAQ;gBAEX,OAAO,EAAE,iBAAiB;IAKtC;;;;;;;OAOG;IACG,kBAAkB,CACtB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,MAAoB,EAC9B,MAAM,GAAE,OAAe,GACtB,OAAO,CAAC,MAAM,CAAC;IAsBlB;;;;;;;OAOG;IACG,eAAe,CACnB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,GAAG,GAAE,MAAyB,EAC9B,MAAM,GAAE,OAAe,GACtB,OAAO,CAAC,MAAM,CAAC;IAsBlB;;;;;;OAMG;IACG,aAAa,CACjB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EACjD,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC;IAwBlB;;;;;;;OAOG;IACG,mBAAmB,CACvB,OAAO,EAAE,eAAe,EAAE,EAC1B,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,OAAe,GACtB,OAAO,CAAC;QACT,SAAS,EAAE,MAAM,CAAA;QACjB,MAAM,EAAE,MAAM,CAAA;KACf,CAAC;IAkCF;;;;OAIG;YACW,iBAAiB;IAyD/B;;;;;OAKG;YACW,qBAAqB;IAsBnC;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;CAa/B;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,GACf,UAAU,GAAG,IAAI,CAInB"}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MiMo-V2 Omni Client
|
|
3
|
+
* Handles video analysis using Xiaomi's MiMo API
|
|
4
|
+
*
|
|
5
|
+
* API Specification (Confirmed):
|
|
6
|
+
* - Base URL: https://api.xiaomimimo.com/v1
|
|
7
|
+
* - Endpoint: https://api.xiaomimimo.com/v1/chat/completions
|
|
8
|
+
* - Auth: api-key header or Authorization: Bearer
|
|
9
|
+
* - Models: mimo-v2-omni (video), mimo-v2-pro, mimo-v2-flash, mimo-v2-tts
|
|
10
|
+
* - Video: URL (max 300MB) or Base64 (max 10MB)
|
|
11
|
+
* - Thinking: enabled by default for omni/pro
|
|
12
|
+
*/
|
|
13
|
+
const MIMO_BASE_URL = 'https://api.xiaomimimo.com/v1/chat/completions';
|
|
14
|
+
const MIMO_MODEL = 'mimo-v2-omni';
|
|
15
|
+
const MIMO_MODEL_PRO = 'mimo-v2-pro';
|
|
16
|
+
const MIMO_MODEL_FLASH = 'mimo-v2-flash';
|
|
17
|
+
/** Default fps for server-side frame extraction on video URL inputs. */
|
|
18
|
+
const MIMO_DEFAULT_FPS = 2.0;
|
|
19
|
+
/** Client-side request timeout in ms. Default: 4 minutes.
|
|
20
|
+
* Override via MIMO_REQUEST_TIMEOUT_MS env var (milliseconds). */
|
|
21
|
+
const MIMO_REQUEST_TIMEOUT_MS = process.env.MIMO_REQUEST_TIMEOUT_MS
|
|
22
|
+
? Number(process.env.MIMO_REQUEST_TIMEOUT_MS)
|
|
23
|
+
: 4 * 60 * 1000;
|
|
24
|
+
const MIMO_TIMEOUT_LABEL = `${Math.round(MIMO_REQUEST_TIMEOUT_MS / 60000)} minutes`;
|
|
25
|
+
export class MimoClient {
|
|
26
|
+
apiKey;
|
|
27
|
+
baseURL;
|
|
28
|
+
constructor(options) {
|
|
29
|
+
this.apiKey = options.apiKey;
|
|
30
|
+
this.baseURL = options.baseURL || MIMO_BASE_URL;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Analyze a video using base64-encoded data
|
|
34
|
+
* @param base64Data - Base64-encoded video data
|
|
35
|
+
* @param prompt - The prompt/question to ask about the video
|
|
36
|
+
* @param mimeType - MIME type of the video (default: 'video/mp4')
|
|
37
|
+
* @param usePro - Use the pro model variant (default: false, uses standard mimo-v2-omni)
|
|
38
|
+
* @returns The AI analysis response
|
|
39
|
+
*/
|
|
40
|
+
async analyzeVideoBase64(base64Data, prompt, mimeType = 'video/mp4', usePro = false) {
|
|
41
|
+
const messages = [
|
|
42
|
+
{
|
|
43
|
+
role: 'user',
|
|
44
|
+
content: [
|
|
45
|
+
{
|
|
46
|
+
type: 'video_url',
|
|
47
|
+
video_url: {
|
|
48
|
+
url: `data:${mimeType};base64,${base64Data}`,
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
type: 'text',
|
|
53
|
+
text: prompt,
|
|
54
|
+
},
|
|
55
|
+
],
|
|
56
|
+
},
|
|
57
|
+
];
|
|
58
|
+
return this.requestChatCompletion(messages, usePro);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Analyze a video from a URL
|
|
62
|
+
* @param url - URL of the video
|
|
63
|
+
* @param prompt - The prompt/question to ask about the video
|
|
64
|
+
* @param fps - Server-side frame extraction rate (default: 2.0)
|
|
65
|
+
* @param usePro - Use the pro model variant (default: false, uses mimo-v2-omni)
|
|
66
|
+
* @returns The AI analysis response
|
|
67
|
+
*/
|
|
68
|
+
async analyzeVideoUrl(url, prompt, fps = MIMO_DEFAULT_FPS, usePro = false) {
|
|
69
|
+
const messages = [
|
|
70
|
+
{
|
|
71
|
+
role: 'user',
|
|
72
|
+
content: [
|
|
73
|
+
{
|
|
74
|
+
type: 'video_url',
|
|
75
|
+
video_url: { url },
|
|
76
|
+
fps,
|
|
77
|
+
media_resolution: 'default',
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
type: 'text',
|
|
81
|
+
text: prompt,
|
|
82
|
+
},
|
|
83
|
+
],
|
|
84
|
+
},
|
|
85
|
+
];
|
|
86
|
+
return this.requestChatCompletion(messages, usePro);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Analyze multiple images (e.g., extracted video frames)
|
|
90
|
+
* Used for the long-video frame-based strategy
|
|
91
|
+
* @param images - Array of image data with MIME types
|
|
92
|
+
* @param prompt - The prompt/question
|
|
93
|
+
* @returns The AI analysis response
|
|
94
|
+
*/
|
|
95
|
+
async analyzeImages(images, prompt) {
|
|
96
|
+
const imageParts = images.map((image) => ({
|
|
97
|
+
type: 'image_url',
|
|
98
|
+
image_url: {
|
|
99
|
+
url: `data:${image.mimeType};base64,${image.data}`,
|
|
100
|
+
},
|
|
101
|
+
}));
|
|
102
|
+
const messages = [
|
|
103
|
+
{
|
|
104
|
+
role: 'user',
|
|
105
|
+
content: [
|
|
106
|
+
...imageParts,
|
|
107
|
+
{
|
|
108
|
+
type: 'text',
|
|
109
|
+
text: prompt,
|
|
110
|
+
},
|
|
111
|
+
],
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
return this.requestChatCompletion(messages, false);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Analyze with thinking mode enabled
|
|
118
|
+
* Returns both the chain-of-thought reasoning and the final answer
|
|
119
|
+
* @param content - Content parts (video or images)
|
|
120
|
+
* @param prompt - The prompt/question
|
|
121
|
+
* @param usePro - Use the pro model variant
|
|
122
|
+
* @returns Object with reasoning and answer
|
|
123
|
+
*/
|
|
124
|
+
async analyzeWithThinking(content, prompt, usePro = false) {
|
|
125
|
+
const messages = [
|
|
126
|
+
{
|
|
127
|
+
role: 'user',
|
|
128
|
+
content: [
|
|
129
|
+
...content,
|
|
130
|
+
{
|
|
131
|
+
type: 'text',
|
|
132
|
+
text: prompt,
|
|
133
|
+
},
|
|
134
|
+
],
|
|
135
|
+
},
|
|
136
|
+
];
|
|
137
|
+
const { data } = await this.requestCompletion({
|
|
138
|
+
model: usePro ? MIMO_MODEL_PRO : MIMO_MODEL,
|
|
139
|
+
messages,
|
|
140
|
+
thinking: { type: 'enabled' },
|
|
141
|
+
});
|
|
142
|
+
if (!data.choices?.[0]?.message) {
|
|
143
|
+
const errorMsg = data.error?.message ?? 'Unknown error';
|
|
144
|
+
throw new Error(`MiMo API request failed: ${errorMsg}`);
|
|
145
|
+
}
|
|
146
|
+
const message = data.choices[0].message;
|
|
147
|
+
return {
|
|
148
|
+
reasoning: message.reasoning_content?.trim() ||
|
|
149
|
+
'No reasoning content returned by MiMo.',
|
|
150
|
+
answer: this.extractTextFromContent(message.content),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Make a chat completion request to the MiMo API
|
|
155
|
+
* @param options - Request options including model, messages, and thinking settings
|
|
156
|
+
* @returns Response object and parsed data
|
|
157
|
+
*/
|
|
158
|
+
async requestCompletion(options) {
|
|
159
|
+
const { model, messages, thinking } = options;
|
|
160
|
+
const controller = new AbortController();
|
|
161
|
+
const timeoutId = setTimeout(() => controller.abort(), MIMO_REQUEST_TIMEOUT_MS);
|
|
162
|
+
try {
|
|
163
|
+
const response = await fetch(this.baseURL, {
|
|
164
|
+
method: 'POST',
|
|
165
|
+
headers: {
|
|
166
|
+
'Content-Type': 'application/json',
|
|
167
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
168
|
+
},
|
|
169
|
+
body: JSON.stringify({
|
|
170
|
+
model,
|
|
171
|
+
messages,
|
|
172
|
+
...(thinking && { thinking }),
|
|
173
|
+
stream: false,
|
|
174
|
+
}),
|
|
175
|
+
signal: controller.signal,
|
|
176
|
+
});
|
|
177
|
+
// Read response as text first to handle non-JSON responses
|
|
178
|
+
const text = await response.text();
|
|
179
|
+
// Try to parse as JSON
|
|
180
|
+
let data;
|
|
181
|
+
try {
|
|
182
|
+
data = JSON.parse(text);
|
|
183
|
+
}
|
|
184
|
+
catch (parseError) {
|
|
185
|
+
throw new Error(`MiMo API returned non-JSON response (HTTP ${response.status}): ${text.substring(0, 200)}`, { cause: parseError });
|
|
186
|
+
}
|
|
187
|
+
return { response, data };
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
if (error instanceof Error && error.name === 'AbortError') {
|
|
191
|
+
throw new Error(`MiMo API request timed out after ${MIMO_TIMEOUT_LABEL}`, { cause: error });
|
|
192
|
+
}
|
|
193
|
+
throw error;
|
|
194
|
+
}
|
|
195
|
+
finally {
|
|
196
|
+
clearTimeout(timeoutId);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Make a chat completion request with standard error handling
|
|
201
|
+
* @param options - Request options
|
|
202
|
+
* @param usePro - Use pro model (for convenience methods)
|
|
203
|
+
* @returns The text response
|
|
204
|
+
*/
|
|
205
|
+
async requestChatCompletion(messages, usePro = false) {
|
|
206
|
+
const { response, data } = await this.requestCompletion({
|
|
207
|
+
model: usePro ? MIMO_MODEL_PRO : MIMO_MODEL,
|
|
208
|
+
messages,
|
|
209
|
+
thinking: { type: 'disabled' },
|
|
210
|
+
});
|
|
211
|
+
if (!response.ok) {
|
|
212
|
+
const errorMessage = data.error?.message ?? response.statusText;
|
|
213
|
+
throw new Error(`MiMo API request failed: ${errorMessage}`);
|
|
214
|
+
}
|
|
215
|
+
if (!data.choices?.[0]?.message?.content) {
|
|
216
|
+
throw new Error('MiMo API returned an empty response');
|
|
217
|
+
}
|
|
218
|
+
return this.extractTextFromContent(data.choices[0].message.content);
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Extract text from message content
|
|
222
|
+
* Handles both string content and array content formats
|
|
223
|
+
* @param content - The message content
|
|
224
|
+
* @returns Extracted text
|
|
225
|
+
*/
|
|
226
|
+
extractTextFromContent(content) {
|
|
227
|
+
if (!content)
|
|
228
|
+
return '';
|
|
229
|
+
if (typeof content === 'string')
|
|
230
|
+
return content.trim();
|
|
231
|
+
// Handle array content format
|
|
232
|
+
return content
|
|
233
|
+
.filter((part) => part.type === 'text' && part.text)
|
|
234
|
+
.map((part) => part.text)
|
|
235
|
+
.join(' ')
|
|
236
|
+
.trim();
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Create a MiMo client if API key is available
|
|
241
|
+
* @param apiKey - MiMo API key (MIMO_API_KEY env var)
|
|
242
|
+
* @param baseURL - Optional custom base URL
|
|
243
|
+
* @returns MiMoClient instance or null if no API key
|
|
244
|
+
*/
|
|
245
|
+
export function createMimoClient(apiKey, baseURL) {
|
|
246
|
+
const key = apiKey || process.env.MIMO_API_KEY;
|
|
247
|
+
if (!key)
|
|
248
|
+
return null;
|
|
249
|
+
return new MimoClient({ apiKey: key, baseURL });
|
|
250
|
+
}
|
|
251
|
+
//# sourceMappingURL=mimoClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mimoClient.js","sourceRoot":"","sources":["../../src/services/mimoClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,aAAa,GAAG,gDAAgD,CAAA;AACtE,MAAM,UAAU,GAAG,cAAc,CAAA;AACjC,MAAM,cAAc,GAAG,aAAa,CAAA;AACpC,MAAM,gBAAgB,GAAG,eAAe,CAAA;AAExC,wEAAwE;AACxE,MAAM,gBAAgB,GAAG,GAAG,CAAA;AAE5B;kEACkE;AAClE,MAAM,uBAAuB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB;IACjE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IAC7C,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAA;AACjB,MAAM,kBAAkB,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAuB,GAAG,KAAK,CAAC,UAAU,CAAA;AAmCnF,MAAM,OAAO,UAAU;IACb,MAAM,CAAQ;IACd,OAAO,CAAQ;IAEvB,YAAY,OAA0B;QACpC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAC5B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,aAAa,CAAA;IACjD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,kBAAkB,CACtB,UAAkB,EAClB,MAAc,EACd,WAAmB,WAAW,EAC9B,SAAkB,KAAK;QAEvB,MAAM,QAAQ,GAAkB;YAC9B;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE;4BACT,GAAG,EAAE,QAAQ,QAAQ,WAAW,UAAU,EAAE;yBAC7C;qBACF;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM;qBACb;iBACF;aACF;SACF,CAAA;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACrD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,eAAe,CACnB,GAAW,EACX,MAAc,EACd,MAAc,gBAAgB,EAC9B,SAAkB,KAAK;QAEvB,MAAM,QAAQ,GAAkB;YAC9B;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE,EAAE,GAAG,EAAE;wBAClB,GAAG;wBACH,gBAAgB,EAAE,SAAS;qBAC5B;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM;qBACb;iBACF;aACF;SACF,CAAA;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACrD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CACjB,MAAiD,EACjD,MAAc;QAEd,MAAM,UAAU,GAAsB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3D,IAAI,EAAE,WAAW;YACjB,SAAS,EAAE;gBACT,GAAG,EAAE,QAAQ,KAAK,CAAC,QAAQ,WAAW,KAAK,CAAC,IAAI,EAAE;aACnD;SACF,CAAC,CAAC,CAAA;QAEH,MAAM,QAAQ,GAAkB;YAC9B;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP,GAAG,UAAU;oBACb;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM;qBACb;iBACF;aACF;SACF,CAAA;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IACpD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,mBAAmB,CACvB,OAA0B,EAC1B,MAAc,EACd,SAAkB,KAAK;QAKvB,MAAM,QAAQ,GAAkB;YAC9B;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP,GAAG,OAAO;oBACV;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM;qBACb;iBACF;aACF;SACF,CAAA;QAED,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC;YAC5C,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU;YAC3C,QAAQ;YACR,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;SAC9B,CAAC,CAAA;QAEF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,eAAe,CAAA;YACvD,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,EAAE,CAAC,CAAA;QACzD,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;QACvC,OAAO;YACL,SAAS,EACP,OAAO,CAAC,iBAAiB,EAAE,IAAI,EAAE;gBACjC,wCAAwC;YAC1C,MAAM,EAAE,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,OAAO,CAAC;SACrD,CAAA;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,iBAAiB,CAAC,OAI/B;QACC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAA;QAE7C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;QACxC,MAAM,SAAS,GAAG,UAAU,CAC1B,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EACxB,uBAAuB,CACxB,CAAA;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE;gBACzC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,QAAQ;oBACR,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;oBAC7B,MAAM,EAAE,KAAK;iBACd,CAAC;gBACF,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAA;YAEF,2DAA2D;YAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YAElC,uBAAuB;YACvB,IAAI,IAAsB,CAAA;YAC1B,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YACzB,CAAC;YAAC,OAAO,UAAU,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CACb,6CAA6C,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAC1F,EAAE,KAAK,EAAE,UAAU,EAAE,CACtB,CAAA;YACH,CAAC;YAED,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,KAAK,CACb,oCAAoC,kBAAkB,EAAE,EACxD,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAA;YACH,CAAC;YACD,MAAM,KAAK,CAAA;QACb,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,SAAS,CAAC,CAAA;QACzB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,qBAAqB,CACjC,QAAuB,EACvB,SAAkB,KAAK;QAEvB,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC;YACtD,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU;YAC3C,QAAQ;YACR,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE;SAC/B,CAAC,CAAA;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAA;YAC/D,MAAM,IAAI,KAAK,CAAC,4BAA4B,YAAY,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;QAED,OAAO,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IACrE,CAAC;IAED;;;;;OAKG;IACK,sBAAsB,CAC5B,OAAqE;QAErE,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAA;QACvB,IAAI,OAAO,OAAO,KAAK,QAAQ;YAAE,OAAO,OAAO,CAAC,IAAI,EAAE,CAAA;QAEtD,8BAA8B;QAC9B,OAAO,OAAO;aACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC;aACnD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;aACxB,IAAI,CAAC,GAAG,CAAC;aACT,IAAI,EAAE,CAAA;IACX,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAe,EACf,OAAgB;IAEhB,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAA;IAC9C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACrB,OAAO,IAAI,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAA;AACjD,CAAC"}
|