video-context-mcp-server 0.11.0-beta → 0.13.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -7
- package/dist/services/providerRouter.d.ts +13 -0
- package/dist/services/providerRouter.d.ts.map +1 -1
- package/dist/services/providerRouter.js +26 -1
- package/dist/services/providerRouter.js.map +1 -1
- package/dist/tools/analyzeVideo.d.ts.map +1 -1
- package/dist/tools/analyzeVideo.js +118 -93
- package/dist/tools/analyzeVideo.js.map +1 -1
- package/dist/tools/extractFrames.d.ts.map +1 -1
- package/dist/tools/extractFrames.js +9 -9
- package/dist/tools/extractFrames.js.map +1 -1
- package/dist/tools/getVideoInfo.d.ts.map +1 -1
- package/dist/tools/getVideoInfo.js +10 -9
- package/dist/tools/getVideoInfo.js.map +1 -1
- package/dist/tools/searchTimestamp.d.ts +1 -11
- package/dist/tools/searchTimestamp.d.ts.map +1 -1
- package/dist/tools/searchTimestamp.js +120 -87
- package/dist/tools/searchTimestamp.js.map +1 -1
- package/dist/tools/summarizeVideo.d.ts.map +1 -1
- package/dist/tools/summarizeVideo.js +128 -124
- package/dist/tools/summarizeVideo.js.map +1 -1
- package/dist/tools/transcribeVideo.d.ts.map +1 -1
- package/dist/tools/transcribeVideo.js +7 -3
- package/dist/tools/transcribeVideo.js.map +1 -1
- package/dist/utils/downloadCache.d.ts +49 -0
- package/dist/utils/downloadCache.d.ts.map +1 -0
- package/dist/utils/downloadCache.js +124 -0
- package/dist/utils/downloadCache.js.map +1 -0
- package/dist/utils/videoUtils.d.ts +54 -3
- package/dist/utils/videoUtils.d.ts.map +1 -1
- package/dist/utils/videoUtils.js +238 -8
- package/dist/utils/videoUtils.js.map +1 -1
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -332,12 +332,35 @@ When a provider's API key is missing, the tool automatically falls back to the n
|
|
|
332
332
|
| `MOONSHOT_API_KEY` | Moonshot AI API key for Kimi K2.5 | Optional (required if using Kimi) |
|
|
333
333
|
| `Z_AI_API_KEY` | Z.AI API key for GLM-4.6V | Optional (required if using GLM) |
|
|
334
334
|
| `VIDEO_MCP_DEFAULT_PROVIDER` | Default video backend (`gemini`, `glm`, `kimi`) | Optional (default: `glm`) |
|
|
335
|
-
| `VIDEO_MCP_MAX_FRAMES` | Max frames for summarization (GLM/Kimi only) | Optional (default:
|
|
335
|
+
| `VIDEO_MCP_MAX_FRAMES` | Max frames for summarization (GLM/Kimi only) | Optional (default: 40; clamped to 5-100) |
|
|
336
336
|
| `DEEPGRAM_API_KEY` | Deepgram API key for `transcribe_video` | Optional (required if using Deepgram) |
|
|
337
337
|
| `ASSEMBLYAI_API_KEY` | AssemblyAI API key for `transcribe_video` | Optional (required if using AssemblyAI) |
|
|
338
338
|
| `GROQ_API_KEY` | Groq API key for Whisper transcription via `transcribe_video` | Optional (required if using Groq) |
|
|
339
339
|
| `AUDIO_MCP_DEFAULT_PROVIDER` | Default audio provider; defaults to `deepgram`. Falls back in order: **deepgram → assemblyai → groq → gemini** when the selected provider's key is unavailable. A fallback notice is included in the response. | Optional (auto-selects from available keys) |
|
|
340
340
|
| `AUDIO_ENHANCE_VIDEO_ANALYSIS` | Controls audio transcript injection into GLM/Kimi `analyze_video`/`summarize_video` prompts. `auto` (default) — transcribes only when the video has a detected audio track; `true` — always attempt transcription; `false` — disabled. A confidence label (`high`/`medium`/`low`) is included in the injected header so the model can weight the transcript appropriately. Gemini is always skipped (handles audio natively). | Optional (default: `auto`) |
|
|
341
|
+
| `VIDEO_MCP_CACHE_TTL_MINUTES` | How long downloaded remote/platform videos are cached (in minutes) across separate tool calls. Cached files live in `os.tmpdir()/video-mcp-cache/` and survive per-call cleanup, avoiding redundant downloads on retries or follow-up questions. Set to `0` to disable caching entirely. | Optional (default: `30`) |
|
|
342
|
+
| `YT_DLP_MAX_RESOLUTION` | Maximum video height in pixels when downloading via yt-dlp (YouTube, Vimeo, TikTok, etc.). Lower values reduce file size and download time. | Optional (default: `720`) |
|
|
343
|
+
| `YT_DLP_PATH` | Path to a custom yt-dlp binary. If unset, uses the binary bundled by `youtube-dl-exec` at install time. | Optional |
|
|
344
|
+
| `YT_DLP_COOKIES_FILE` | Path to a Netscape-format cookies file for authenticated yt-dlp downloads (e.g. age-restricted or private YouTube videos). | Optional |
|
|
345
|
+
|
|
346
|
+
> **Tip: install yt-dlp system-wide for faster startup.** The `youtube-dl-exec` package bundles a yt-dlp binary downloaded at `npm install` time, which adds ~30 MB to the install footprint. If you use platform video URLs regularly, it's lighter to install yt-dlp directly and point to it:
|
|
347
|
+
>
|
|
348
|
+
> ```bash
|
|
349
|
+
> # macOS
|
|
350
|
+
> brew install yt-dlp
|
|
351
|
+
>
|
|
352
|
+
> # Windows (winget)
|
|
353
|
+
> winget install yt-dlp
|
|
354
|
+
>
|
|
355
|
+
> # pip (cross-platform)
|
|
356
|
+
> pip install yt-dlp
|
|
357
|
+
> ```
|
|
358
|
+
>
|
|
359
|
+
> Then set `YT_DLP_PATH` to the installed binary (e.g. `/opt/homebrew/bin/yt-dlp` or `C:/Users/you/AppData/Local/Microsoft/WinGet/Packages/yt-dlp.yt-dlp_Microsoft.Winget.Source_8wekyb3d8bbwe/yt-dlp.exe`). To skip the bundled download entirely at install time, set `YOUTUBE_DL_SKIP_DOWNLOAD=true` in your environment before running `npm install`:
|
|
360
|
+
>
|
|
361
|
+
> ```bash
|
|
362
|
+
> YOUTUBE_DL_SKIP_DOWNLOAD=true npm install -g video-context-mcp-server
|
|
363
|
+
> ```
|
|
341
364
|
|
|
342
365
|
### Example Configuration
|
|
343
366
|
|
|
@@ -355,7 +378,10 @@ When a provider's API key is missing, the tool automatically falls back to the n
|
|
|
355
378
|
"ASSEMBLYAI_API_KEY": "your-assemblyai-key",
|
|
356
379
|
"GROQ_API_KEY": "your-groq-key",
|
|
357
380
|
"VIDEO_MCP_DEFAULT_PROVIDER": "glm",
|
|
358
|
-
"
|
|
381
|
+
"VIDEO_MCP_MAX_FRAMES": "40",
|
|
382
|
+
"AUDIO_ENHANCE_VIDEO_ANALYSIS": "auto",
|
|
383
|
+
"VIDEO_MCP_CACHE_TTL_MINUTES": "30",
|
|
384
|
+
"YT_DLP_MAX_RESOLUTION": "720"
|
|
359
385
|
}
|
|
360
386
|
}
|
|
361
387
|
}
|
|
@@ -363,6 +389,8 @@ When a provider's API key is missing, the tool automatically falls back to the n
|
|
|
363
389
|
```
|
|
364
390
|
|
|
365
391
|
> **Note:** `VIDEO_MCP_MAX_FRAMES` only applies when using **GLM or Kimi** as the provider. Gemini uploads the full video natively and ignores this setting. Add it to `env` only if you are running with `VIDEO_MCP_DEFAULT_PROVIDER=gemini`.
|
|
392
|
+
>
|
|
393
|
+
> **Note:** `YT_DLP_PATH` and `YT_DLP_COOKIES_FILE` are only needed if you want to use a custom yt-dlp binary or authenticate downloads of private/age-restricted videos. Omit them for standard use.
|
|
366
394
|
|
|
367
395
|
## Development
|
|
368
396
|
|
|
@@ -407,15 +435,16 @@ npm run ltfb
|
|
|
407
435
|
video-mcp/
|
|
408
436
|
├── src/
|
|
409
437
|
│ ├── index.ts # MCP server entry point
|
|
410
|
-
│ ├── tools/ # MCP tool implementations
|
|
411
|
-
│
|
|
412
|
-
│ ├── services/ # Backend clients (Kimi, GLM, Gemini, ffmpeg)
|
|
438
|
+
│ ├── tools/ # MCP tool implementations (6 tools)
|
|
439
|
+
│ ├── services/ # Backend clients (Kimi, GLM, Gemini, ffmpeg, audioRouter)
|
|
413
440
|
│ │ └── audio/ # Audio provider clients (Deepgram, AssemblyAI, Groq)
|
|
414
|
-
│ └── utils/ # Helpers (
|
|
441
|
+
│ └── utils/ # Helpers (tempFiles, base64, videoUtils, downloadCache, audioUtils, logger)
|
|
415
442
|
├── .vscode/
|
|
416
443
|
│ └── mcp.json # VS Code MCP configuration
|
|
417
444
|
├── docs/
|
|
418
|
-
│
|
|
445
|
+
│ ├── features/ # Feature design and progress docs
|
|
446
|
+
│ ├── reviews/ # Codebase review notes
|
|
447
|
+
│ └── plan-and-progress.md # Implementation plan and iteration history
|
|
419
448
|
└── .github/
|
|
420
449
|
└── copilot-instructions.md # Copilot AI assistant guidelines
|
|
421
450
|
```
|
|
@@ -33,6 +33,19 @@ export declare function resolveProviderWithFallback(provider: VideoProvider, has
|
|
|
33
33
|
* Get the default provider from environment variable
|
|
34
34
|
*/
|
|
35
35
|
export declare function getDefaultProvider(): VideoProvider;
|
|
36
|
+
/**
|
|
37
|
+
* Returns an ordered list of providers to attempt for a given starting provider,
|
|
38
|
+
* filtered to only those whose API keys are available.
|
|
39
|
+
*
|
|
40
|
+
* Fallback order (mirrors the key-check convention in resolveProviderWithFallback):
|
|
41
|
+
* glm → glm, kimi, gemini (filtered to those with keys)
|
|
42
|
+
* kimi → kimi, gemini (no kimi → glm fallback)
|
|
43
|
+
* gemini → gemini (last resort, no fallback)
|
|
44
|
+
*
|
|
45
|
+
* This is used for runtime error-based fallback: if the first provider's API call
|
|
46
|
+
* throws (overloaded, timeout, 5xx, etc.), the caller retries with the next entry.
|
|
47
|
+
*/
|
|
48
|
+
export declare function getRuntimeFallbackChain(provider: VideoProvider, hasKimiKey: boolean, hasGLMKey: boolean, hasGeminiKey?: boolean): VideoProvider[];
|
|
36
49
|
/**
|
|
37
50
|
* Get the maximum number of frames to extract for summarization
|
|
38
51
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providerRouter.d.ts","sourceRoot":"","sources":["../../src/services/providerRouter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAA;AAErD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,aAAa,EACvB,UAAU,EAAE,OAAO,EACnB,SAAS,EAAE,OAAO,EAClB,YAAY,GAAE,OAAe,GAC5B,aAAa,CAiBf;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,aAAa,CAAA;IACvB,YAAY,CAAC,EAAE,aAAa,CAAA;CAC7B,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,aAAa,EACvB,UAAU,EAAE,OAAO,EACnB,SAAS,EAAE,OAAO,EAClB,YAAY,GAAE,OAAe,GAC5B,cAAc,CAgBhB;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,aAAa,CAUlD;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAGrC"}
|
|
1
|
+
{"version":3,"file":"providerRouter.d.ts","sourceRoot":"","sources":["../../src/services/providerRouter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAA;AAErD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,aAAa,EACvB,UAAU,EAAE,OAAO,EACnB,SAAS,EAAE,OAAO,EAClB,YAAY,GAAE,OAAe,GAC5B,aAAa,CAiBf;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,aAAa,CAAA;IACvB,YAAY,CAAC,EAAE,aAAa,CAAA;CAC7B,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,aAAa,EACvB,UAAU,EAAE,OAAO,EACnB,SAAS,EAAE,OAAO,EAClB,YAAY,GAAE,OAAe,GAC5B,cAAc,CAgBhB;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,aAAa,CAUlD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,aAAa,EACvB,UAAU,EAAE,OAAO,EACnB,SAAS,EAAE,OAAO,EAClB,YAAY,GAAE,OAAe,GAC5B,aAAa,EAAE,CAejB;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAGrC"}
|
|
@@ -54,11 +54,36 @@ export function getDefaultProvider() {
|
|
|
54
54
|
}
|
|
55
55
|
return 'glm';
|
|
56
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Returns an ordered list of providers to attempt for a given starting provider,
|
|
59
|
+
* filtered to only those whose API keys are available.
|
|
60
|
+
*
|
|
61
|
+
* Fallback order (mirrors the key-check convention in resolveProviderWithFallback):
|
|
62
|
+
* glm → glm, kimi, gemini (filtered to those with keys)
|
|
63
|
+
* kimi → kimi, gemini (no kimi → glm fallback)
|
|
64
|
+
* gemini → gemini (last resort, no fallback)
|
|
65
|
+
*
|
|
66
|
+
* This is used for runtime error-based fallback: if the first provider's API call
|
|
67
|
+
* throws (overloaded, timeout, 5xx, etc.), the caller retries with the next entry.
|
|
68
|
+
*/
|
|
69
|
+
export function getRuntimeFallbackChain(provider, hasKimiKey, hasGLMKey, hasGeminiKey = false) {
|
|
70
|
+
const candidates = provider === 'glm'
|
|
71
|
+
? ['glm', 'kimi', 'gemini']
|
|
72
|
+
: provider === 'kimi'
|
|
73
|
+
? ['kimi', 'gemini']
|
|
74
|
+
: ['gemini'];
|
|
75
|
+
const keyMap = {
|
|
76
|
+
glm: hasGLMKey,
|
|
77
|
+
kimi: hasKimiKey,
|
|
78
|
+
gemini: hasGeminiKey,
|
|
79
|
+
};
|
|
80
|
+
return candidates.filter((p) => keyMap[p]);
|
|
81
|
+
}
|
|
57
82
|
/**
|
|
58
83
|
* Get the maximum number of frames to extract for summarization
|
|
59
84
|
*/
|
|
60
85
|
export function getMaxFrames() {
|
|
61
|
-
const envMaxFrames = parseInt(process.env.VIDEO_MCP_MAX_FRAMES || '
|
|
86
|
+
const envMaxFrames = parseInt(process.env.VIDEO_MCP_MAX_FRAMES || '40', 10);
|
|
62
87
|
return Math.max(5, Math.min(100, envMaxFrames)); // Clamp between 5 and 100
|
|
63
88
|
}
|
|
64
89
|
//# sourceMappingURL=providerRouter.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providerRouter.js","sourceRoot":"","sources":["../../src/services/providerRouter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAuB,EACvB,UAAmB,EACnB,SAAkB,EAClB,eAAwB,KAAK;IAE7B,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CACb,+DAA+D,CAChE,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,KAAK,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,oEAAoE,CACrE,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,+DAA+D,CAChE,CAAA;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAYD;;;;;;GAMG;AACH,MAAM,UAAU,2BAA2B,CACzC,QAAuB,EACvB,UAAmB,EACnB,SAAkB,EAClB,eAAwB,KAAK;IAE7B,IAAI,QAAQ,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,UAAU;YAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAA;QAChE,IAAI,YAAY;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,CAAA;IACtE,CAAC;IACD,IAAI,QAAQ,KAAK,MAAM,IAAI,CAAC,UAAU,IAAI,YAAY,EAAE,CAAC;QACvD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,CAAA;IACrD,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAC9B,QAAQ,EACR,UAAU,EACV,SAAS,EACT,YAAY,CACb,CAAA;IACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAA;IAC1D,IACE,WAAW,KAAK,KAAK;QACrB,WAAW,KAAK,MAAM;QACtB,WAAW,KAAK,QAAQ,EACxB,CAAC;QACD,OAAO,WAAW,CAAA;IACpB,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,IAAI,EAAE,EAAE,CAAC,CAAA;IAC3E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAA,CAAC,0BAA0B;AAC5E,CAAC"}
|
|
1
|
+
{"version":3,"file":"providerRouter.js","sourceRoot":"","sources":["../../src/services/providerRouter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAuB,EACvB,UAAmB,EACnB,SAAkB,EAClB,eAAwB,KAAK;IAE7B,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CACb,+DAA+D,CAChE,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,KAAK,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,oEAAoE,CACrE,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,+DAA+D,CAChE,CAAA;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAYD;;;;;;GAMG;AACH,MAAM,UAAU,2BAA2B,CACzC,QAAuB,EACvB,UAAmB,EACnB,SAAkB,EAClB,eAAwB,KAAK;IAE7B,IAAI,QAAQ,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,UAAU;YAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAA;QAChE,IAAI,YAAY;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,CAAA;IACtE,CAAC;IACD,IAAI,QAAQ,KAAK,MAAM,IAAI,CAAC,UAAU,IAAI,YAAY,EAAE,CAAC;QACvD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,CAAA;IACrD,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAC9B,QAAQ,EACR,UAAU,EACV,SAAS,EACT,YAAY,CACb,CAAA;IACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAA;IAC1D,IACE,WAAW,KAAK,KAAK;QACrB,WAAW,KAAK,MAAM;QACtB,WAAW,KAAK,QAAQ,EACxB,CAAC;QACD,OAAO,WAAW,CAAA;IACpB,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,uBAAuB,CACrC,QAAuB,EACvB,UAAmB,EACnB,SAAkB,EAClB,eAAwB,KAAK;IAE7B,MAAM,UAAU,GACd,QAAQ,KAAK,KAAK;QAChB,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC;QAC3B,CAAC,CAAC,QAAQ,KAAK,MAAM;YACnB,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC;YACpB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;IAElB,MAAM,MAAM,GAAmC;QAC7C,GAAG,EAAE,SAAS;QACd,IAAI,EAAE,UAAU;QAChB,MAAM,EAAE,YAAY;KACrB,CAAA;IAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,IAAI,EAAE,EAAE,CAAC,CAAA;IAC3E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAA,CAAC,0BAA0B;AAC5E,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyzeVideo.d.ts","sourceRoot":"","sources":["../../src/tools/analyzeVideo.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"analyzeVideo.d.ts","sourceRoot":"","sources":["../../src/tools/analyzeVideo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA4BH,eAAO,MAAM,gBAAgB,GAAU,QAAQ,GAAG;;;;;;;;;;;;EAkMjD,CAAA"}
|
|
@@ -7,121 +7,147 @@ import { createReadStream } from 'fs';
|
|
|
7
7
|
import { createGLMClient } from '../services/glmClient.js';
|
|
8
8
|
import { createKimiClient } from '../services/kimiClient.js';
|
|
9
9
|
import { createGeminiClient } from '../services/geminiClient.js';
|
|
10
|
-
import { getDefaultProvider,
|
|
10
|
+
import { getDefaultProvider, getRuntimeFallbackChain, } from '../services/providerRouter.js';
|
|
11
11
|
import { bufferToBase64String } from '../utils/base64.js';
|
|
12
12
|
import { createTempDir, cleanupTempDir } from '../utils/tempFiles.js';
|
|
13
|
-
import { isRemoteUrl, normalizeVideoPath, downloadVideoToTemp, } from '../utils/videoUtils.js';
|
|
13
|
+
import { isPlatformUrl, isRemoteUrl, normalizeVideoPath, downloadVideoToTemp, downloadWithYtDlp, } from '../utils/videoUtils.js';
|
|
14
14
|
import { logProgress } from '../utils/logger.js';
|
|
15
15
|
import { getAudioEnhancementMode, tryExtractAndTranscribeAudio, buildPromptWithTranscript, } from '../utils/audioUtils.js';
|
|
16
16
|
export const analyzeVideoTool = async (params) => {
|
|
17
17
|
let tempDir = null;
|
|
18
18
|
try {
|
|
19
19
|
const { videoPath, question } = params;
|
|
20
|
-
const
|
|
20
|
+
const requestedProvider = params.provider ?? getDefaultProvider();
|
|
21
21
|
const glmClient = createGLMClient();
|
|
22
22
|
const kimiClient = createKimiClient();
|
|
23
23
|
const geminiClient = createGeminiClient();
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
// ── Step 1: Resolve platform URLs immediately (always needs yt-dlp) ──────
|
|
25
|
+
// For plain remote URLs and local paths the download is deferred until we
|
|
26
|
+
// know which provider will be used (GLM can analyse remote URLs directly).
|
|
27
|
+
const isPlatform = isPlatformUrl(videoPath);
|
|
28
|
+
const isRemote = !isPlatform && isRemoteUrl(videoPath);
|
|
29
|
+
let localPath = null;
|
|
30
|
+
let effectivePath; // local file path OR original remote URL for GLM
|
|
31
|
+
if (isPlatform) {
|
|
32
|
+
// Platform URLs must always be resolved to a local file first.
|
|
33
|
+
// downloadWithYtDlp() caches the result so a second call is instant.
|
|
34
|
+
tempDir = await createTempDir(); // used for audio later
|
|
35
|
+
localPath = await downloadWithYtDlp(videoPath, tempDir);
|
|
36
|
+
effectivePath = localPath;
|
|
37
|
+
}
|
|
38
|
+
else if (!isRemote) {
|
|
39
|
+
// Local file (or file:// URI)
|
|
40
|
+
effectivePath = normalizeVideoPath(videoPath);
|
|
41
|
+
localPath = effectivePath;
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
// Plain http(s) remote URL — download deferred (GLM can use directly)
|
|
45
|
+
effectivePath = videoPath;
|
|
46
|
+
localPath = null;
|
|
47
|
+
}
|
|
48
|
+
// ── Step 2: Build the provider fallback chain ─────────────────────────────
|
|
49
|
+
const chain = getRuntimeFallbackChain(requestedProvider, Boolean(kimiClient), Boolean(glmClient), Boolean(geminiClient));
|
|
50
|
+
if (chain.length === 0) {
|
|
51
|
+
throw new Error('No video AI providers are configured. Set Z_AI_API_KEY, MOONSHOT_API_KEY, or GEMINI_API_KEY.');
|
|
52
|
+
}
|
|
53
|
+
// Notify about key-based fallback at the start of the chain
|
|
54
|
+
const firstProvider = chain[0];
|
|
55
|
+
if (firstProvider !== requestedProvider) {
|
|
56
|
+
await logProgress(`Provider ${requestedProvider} unavailable (no API key), using ${firstProvider}`);
|
|
57
|
+
}
|
|
58
|
+
// ── Step 3: Retry loop over providers ────────────────────────────────────
|
|
59
|
+
// State shared across retries (computed once, reused on fallback)
|
|
60
|
+
let enhancedQuestion = null; // lazy: first non-Gemini attempt
|
|
26
61
|
let answer;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
62
|
+
let selectedProvider = firstProvider;
|
|
63
|
+
let lastError = '';
|
|
64
|
+
const runtimeFallbacks = []; // providers skipped due to runtime errors
|
|
65
|
+
for (let i = 0; i < chain.length; i++) {
|
|
66
|
+
const p = chain[i];
|
|
67
|
+
if (i > 0) {
|
|
68
|
+
runtimeFallbacks.push(chain[i - 1]);
|
|
69
|
+
await logProgress(`${chain[i - 1]} failed (${lastError}), retrying with ${p}...`);
|
|
33
70
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
71
|
+
try {
|
|
72
|
+
selectedProvider = p;
|
|
73
|
+
// Ensure a local file path for providers that need it.
|
|
74
|
+
// GLM gets a special optimisation for non-platform remote URLs: it can
|
|
75
|
+
// analyse the URL directly without downloading.
|
|
76
|
+
const isGlmDirectUrl = p === 'glm' && isRemote && !localPath;
|
|
77
|
+
if (!isGlmDirectUrl && !localPath) {
|
|
78
|
+
if (!tempDir)
|
|
79
|
+
tempDir = await createTempDir();
|
|
80
|
+
await logProgress('Downloading video...');
|
|
81
|
+
localPath = await downloadVideoToTemp(effectivePath, tempDir);
|
|
37
82
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
if (!glmClient) {
|
|
50
|
-
throw new Error('GLM provider is required for URL video analysis, but Z_AI_API_KEY is not configured.');
|
|
83
|
+
// Audio transcript injection (non-Gemini only; computed once and reused)
|
|
84
|
+
if (p !== 'gemini' && !isGlmDirectUrl && enhancedQuestion === null) {
|
|
85
|
+
if (getAudioEnhancementMode() !== 'false') {
|
|
86
|
+
if (!tempDir)
|
|
87
|
+
tempDir = await createTempDir();
|
|
88
|
+
const result = await tryExtractAndTranscribeAudio(localPath, tempDir);
|
|
89
|
+
enhancedQuestion = buildPromptWithTranscript(question, result);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
enhancedQuestion = question;
|
|
93
|
+
}
|
|
51
94
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
if (
|
|
57
|
-
|
|
95
|
+
const questionToUse = p === 'gemini' || isGlmDirectUrl
|
|
96
|
+
? question
|
|
97
|
+
: (enhancedQuestion ?? question);
|
|
98
|
+
// ── Provider-specific AI call ───────────────────────────────────────
|
|
99
|
+
if (p === 'gemini') {
|
|
100
|
+
if (!geminiClient)
|
|
101
|
+
throw new Error('Gemini client unavailable. Set GEMINI_API_KEY.');
|
|
102
|
+
await logProgress('Uploading to Gemini...');
|
|
103
|
+
const uploaded = await geminiClient.uploadVideo(localPath);
|
|
104
|
+
await logProgress('Waiting for Gemini to process video...');
|
|
105
|
+
await geminiClient.waitForFileProcessing(uploaded.name);
|
|
106
|
+
await logProgress('Analyzing video...');
|
|
107
|
+
answer = await geminiClient.analyzeVideo(uploaded, question);
|
|
58
108
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
109
|
+
else if (p === 'glm') {
|
|
110
|
+
if (!glmClient)
|
|
111
|
+
throw new Error('GLM client unavailable. Set Z_AI_API_KEY.');
|
|
112
|
+
await logProgress('Analyzing video...');
|
|
113
|
+
if (isGlmDirectUrl) {
|
|
114
|
+
answer = await glmClient.analyzeVideoUrl(effectivePath, questionToUse);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
const videoBuffer = await fs.readFile(localPath);
|
|
118
|
+
answer = await glmClient.analyzeVideoBase64(bufferToBase64String(videoBuffer), questionToUse);
|
|
119
|
+
}
|
|
66
120
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
}
|
|
76
|
-
else {
|
|
77
|
-
const resolvedPath = normalizeVideoPath(videoPath);
|
|
78
|
-
const resolved = resolveProviderWithFallback(provider, Boolean(kimiClient), Boolean(glmClient), Boolean(geminiClient));
|
|
79
|
-
selectedProvider = resolved.provider;
|
|
80
|
-
if (resolved.fallbackFrom) {
|
|
81
|
-
fallbackNotice = ` (fell back from ${resolved.fallbackFrom})`;
|
|
82
|
-
await logProgress(`Provider ${resolved.fallbackFrom} unavailable, using ${resolved.provider}`);
|
|
83
|
-
}
|
|
84
|
-
// Inject audio transcript into the question for GLM/Kimi (Gemini handles audio natively)
|
|
85
|
-
let enhancedQuestion = question;
|
|
86
|
-
if (selectedProvider !== 'gemini' &&
|
|
87
|
-
getAudioEnhancementMode() !== 'false') {
|
|
88
|
-
if (!tempDir)
|
|
89
|
-
tempDir = await createTempDir();
|
|
90
|
-
const result = await tryExtractAndTranscribeAudio(resolvedPath, tempDir);
|
|
91
|
-
enhancedQuestion = buildPromptWithTranscript(question, result);
|
|
92
|
-
}
|
|
93
|
-
if (selectedProvider === 'gemini') {
|
|
94
|
-
if (!geminiClient) {
|
|
95
|
-
throw new Error('Gemini client is unavailable. Set GEMINI_API_KEY.');
|
|
96
|
-
}
|
|
97
|
-
await logProgress('Uploading to Gemini...');
|
|
98
|
-
const uploadedVideo = await geminiClient.uploadVideo(resolvedPath);
|
|
99
|
-
await logProgress('Waiting for Gemini to process video...');
|
|
100
|
-
await geminiClient.waitForFileProcessing(uploadedVideo.name);
|
|
101
|
-
await logProgress('Analyzing video...');
|
|
102
|
-
answer = await geminiClient.analyzeVideo(uploadedVideo, question);
|
|
103
|
-
}
|
|
104
|
-
else if (selectedProvider === 'glm') {
|
|
105
|
-
if (!glmClient) {
|
|
106
|
-
throw new Error('GLM client is unavailable. Set Z_AI_API_KEY.');
|
|
121
|
+
else {
|
|
122
|
+
// kimi
|
|
123
|
+
if (!kimiClient)
|
|
124
|
+
throw new Error('Kimi client unavailable. Set MOONSHOT_API_KEY.');
|
|
125
|
+
await logProgress('Uploading to Kimi...');
|
|
126
|
+
const uploaded = await kimiClient.uploadVideo(createReadStream(localPath));
|
|
127
|
+
await logProgress('Analyzing video...');
|
|
128
|
+
answer = await kimiClient.analyzeVideo(uploaded.id, questionToUse);
|
|
107
129
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
answer = await glmClient.analyzeVideoBase64(bufferToBase64String(videoBuffer), enhancedQuestion);
|
|
130
|
+
// SUCCESS — break out of loop
|
|
131
|
+
break;
|
|
111
132
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
133
|
+
catch (err) {
|
|
134
|
+
lastError = err instanceof Error ? err.message : String(err);
|
|
135
|
+
if (i === chain.length - 1) {
|
|
136
|
+
// All providers exhausted — surface the last error
|
|
137
|
+
throw err;
|
|
115
138
|
}
|
|
116
|
-
|
|
117
|
-
const uploadedVideo = await kimiClient.uploadVideo(createReadStream(resolvedPath));
|
|
118
|
-
await logProgress('Analyzing video...');
|
|
119
|
-
answer = await kimiClient.analyzeVideo(uploadedVideo.id, enhancedQuestion);
|
|
139
|
+
// Continue to the next provider
|
|
120
140
|
}
|
|
121
141
|
}
|
|
122
|
-
if (tempDir)
|
|
142
|
+
if (tempDir)
|
|
123
143
|
await cleanupTempDir(tempDir);
|
|
144
|
+
// ── Build fallback notice ─────────────────────────────────────────────────
|
|
145
|
+
const parts = [];
|
|
146
|
+
if (firstProvider !== requestedProvider) {
|
|
147
|
+
parts.push(`${requestedProvider} (no API key)`);
|
|
124
148
|
}
|
|
149
|
+
runtimeFallbacks.forEach((p) => parts.push(`${p} (runtime error)`));
|
|
150
|
+
const fallbackNotice = parts.length > 0 ? ` (fell back from: ${parts.join(', ')})` : '';
|
|
125
151
|
return {
|
|
126
152
|
content: [
|
|
127
153
|
{
|
|
@@ -136,9 +162,8 @@ export const analyzeVideoTool = async (params) => {
|
|
|
136
162
|
};
|
|
137
163
|
}
|
|
138
164
|
catch (error) {
|
|
139
|
-
if (tempDir)
|
|
165
|
+
if (tempDir)
|
|
140
166
|
await cleanupTempDir(tempDir).catch(() => { });
|
|
141
|
-
}
|
|
142
167
|
return {
|
|
143
168
|
isError: true,
|
|
144
169
|
content: [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyzeVideo.js","sourceRoot":"","sources":["../../src/tools/analyzeVideo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChE,OAAO,EACL,kBAAkB,EAClB,
|
|
1
|
+
{"version":3,"file":"analyzeVideo.js","sourceRoot":"","sources":["../../src/tools/analyzeVideo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChE,OAAO,EACL,kBAAkB,EAClB,uBAAuB,GAExB,MAAM,+BAA+B,CAAA;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACrE,OAAO,EACL,aAAa,EACb,WAAW,EACX,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAChD,OAAO,EACL,uBAAuB,EACvB,4BAA4B,EAC5B,yBAAyB,GAC1B,MAAM,wBAAwB,CAAA;AAE/B,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAW,EAAE,EAAE;IACpD,IAAI,OAAO,GAAkB,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAA;QACtC,MAAM,iBAAiB,GACrB,MAAM,CAAC,QAAQ,IAAI,kBAAkB,EAAE,CAAA;QAEzC,MAAM,SAAS,GAAG,eAAe,EAAE,CAAA;QACnC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAA;QACrC,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAA;QAEzC,4EAA4E;QAC5E,0EAA0E;QAC1E,2EAA2E;QAC3E,MAAM,UAAU,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;QAC3C,MAAM,QAAQ,GAAG,CAAC,UAAU,IAAI,WAAW,CAAC,SAAS,CAAC,CAAA;QAEtD,IAAI,SAAS,GAAkB,IAAI,CAAA;QACnC,IAAI,aAAqB,CAAA,CAAC,iDAAiD;QAE3E,IAAI,UAAU,EAAE,CAAC;YACf,+DAA+D;YAC/D,qEAAqE;YACrE,OAAO,GAAG,MAAM,aAAa,EAAE,CAAA,CAAC,uBAAuB;YACvD,SAAS,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YACvD,aAAa,GAAG,SAAS,CAAA;QAC3B,CAAC;aAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACrB,8BAA8B;YAC9B,aAAa,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;YAC7C,SAAS,GAAG,aAAa,CAAA;QAC3B,CAAC;aAAM,CAAC;YACN,sEAAsE;YACtE,aAAa,GAAG,SAAS,CAAA;YACzB,SAAS,GAAG,IAAI,CAAA;QAClB,CAAC;QAED,6EAA6E;QAC7E,MAAM,KAAK,GAAG,uBAAuB,CACnC,iBAAiB,EACjB,OAAO,CAAC,UAAU,CAAC,EACnB,OAAO,CAAC,SAAS,CAAC,EAClB,OAAO,CAAC,YAAY,CAAC,CACtB,CAAA;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,8FAA8F,CAC/F,CAAA;QACH,CAAC;QAED,4DAA4D;QAC5D,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAC9B,IAAI,aAAa,KAAK,iBAAiB,EAAE,CAAC;YACxC,MAAM,WAAW,CACf,YAAY,iBAAiB,oCAAoC,aAAa,EAAE,CACjF,CAAA;QACH,CAAC;QAED,4EAA4E;QAC5E,kEAAkE;QAClE,IAAI,gBAAgB,GAAkB,IAAI,CAAA,CAAC,iCAAiC;QAC5E,IAAI,MAA0B,CAAA;QAC9B,IAAI,gBAAgB,GAAkB,aAAa,CAAA;QACnD,IAAI,SAAS,GAAG,EAAE,CAAA;QAClB,MAAM,gBAAgB,GAAa,EAAE,CAAA,CAAC,0CAA0C;QAEhF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YAElB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACV,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBACnC,MAAM,WAAW,CACf,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,SAAS,oBAAoB,CAAC,KAAK,CAC/D,CAAA;YACH,CAAC;YAED,IAAI,CAAC;gBACH,gBAAgB,GAAG,CAAC,CAAA;gBAEpB,uDAAuD;gBACvD,uEAAuE;gBACvE,gDAAgD;gBAChD,MAAM,cAAc,GAAG,CAAC,KAAK,KAAK,IAAI,QAAQ,IAAI,CAAC,SAAS,CAAA;gBAE5D,IAAI,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE,CAAC;oBAClC,IAAI,CAAC,OAAO;wBAAE,OAAO,GAAG,MAAM,aAAa,EAAE,CAAA;oBAC7C,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;oBACzC,SAAS,GAAG,MAAM,mBAAmB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAA;gBAC/D,CAAC;gBAED,yEAAyE;gBACzE,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,cAAc,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;oBACnE,IAAI,uBAAuB,EAAE,KAAK,OAAO,EAAE,CAAC;wBAC1C,IAAI,CAAC,OAAO;4BAAE,OAAO,GAAG,MAAM,aAAa,EAAE,CAAA;wBAC7C,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAC/C,SAAU,EACV,OAAO,CACR,CAAA;wBACD,gBAAgB,GAAG,yBAAyB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;oBAChE,CAAC;yBAAM,CAAC;wBACN,gBAAgB,GAAG,QAAQ,CAAA;oBAC7B,CAAC;gBACH,CAAC;gBAED,MAAM,aAAa,GACjB,CAAC,KAAK,QAAQ,IAAI,cAAc;oBAC9B,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,CAAC,gBAAgB,IAAI,QAAQ,CAAC,CAAA;gBAEpC,uEAAuE;gBACvE,IAAI,CAAC,KAAK,QAAQ,EAAE,CAAC;oBACnB,IAAI,CAAC,YAAY;wBACf,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;oBACnE,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;oBAC3C,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,SAAU,CAAC,CAAA;oBAC3D,MAAM,WAAW,CAAC,wCAAwC,CAAC,CAAA;oBAC3D,MAAM,YAAY,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;oBACvD,MAAM,WAAW,CAAC,oBAAoB,CAAC,CAAA;oBACvC,MAAM,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;gBAC9D,CAAC;qBAAM,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC,SAAS;wBACZ,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;oBAC9D,MAAM,WAAW,CAAC,oBAAoB,CAAC,CAAA;oBACvC,IAAI,cAAc,EAAE,CAAC;wBACnB,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CACtC,aAAa,EACb,aAAa,CACd,CAAA;oBACH,CAAC;yBAAM,CAAC;wBACN,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAU,CAAC,CAAA;wBACjD,MAAM,GAAG,MAAM,SAAS,CAAC,kBAAkB,CACzC,oBAAoB,CAAC,WAAW,CAAC,EACjC,aAAa,CACd,CAAA;oBACH,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,OAAO;oBACP,IAAI,CAAC,UAAU;wBACb,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;oBACnE,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;oBACzC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAC3C,gBAAgB,CAAC,SAAU,CAAC,CAC7B,CAAA;oBACD,MAAM,WAAW,CAAC,oBAAoB,CAAC,CAAA;oBACvC,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;gBACpE,CAAC;gBAED,8BAA8B;gBAC9B,MAAK;YACP,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAC5D,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3B,mDAAmD;oBACnD,MAAM,GAAG,CAAA;gBACX,CAAC;gBACD,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;QAE1C,6EAA6E;QAC7E,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,IAAI,aAAa,KAAK,iBAAiB,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,GAAG,iBAAiB,eAAe,CAAC,CAAA;QACjD,CAAC;QACD,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAA;QACnE,MAAM,cAAc,GAClB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;QAElE,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,MAAO;iBACd;gBACD;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,kBAAkB,gBAAgB,GAAG,cAAc,EAAE;iBAC5D;aACF;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC1D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBACzF;aACF;SACF,CAAA;IACH,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractFrames.d.ts","sourceRoot":"","sources":["../../src/tools/extractFrames.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"extractFrames.d.ts","sourceRoot":"","sources":["../../src/tools/extractFrames.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAaH,eAAO,MAAM,iBAAiB,GAAU,QAAQ,GAAG;;;;;;;;;;;;;;;;EA8ElD,CAAA"}
|
|
@@ -4,21 +4,21 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { extractFrameAt, extractFramesAtInterval, extractFramesAtTimestamps, extractFramesEvenly, } from '../services/ffmpeg.js';
|
|
6
6
|
import { bufferToBase64String } from '../utils/base64.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
import { cleanupTempDir } from '../utils/tempFiles.js';
|
|
8
|
+
import { resolveVideoInput } from '../utils/videoUtils.js';
|
|
9
9
|
import { logProgress } from '../utils/logger.js';
|
|
10
10
|
export const extractFramesTool = async (params) => {
|
|
11
11
|
let tempDir = null;
|
|
12
12
|
try {
|
|
13
13
|
const { videoPath, mode, count, intervalSec, timestamps } = params;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
const resolved = await resolveVideoInput(videoPath);
|
|
15
|
+
const resolvedPath = resolved.resolvedPath;
|
|
16
|
+
tempDir = resolved.tempDir;
|
|
17
|
+
if (resolved.source === 'platform') {
|
|
18
|
+
await logProgress('Downloaded from platform via yt-dlp.');
|
|
19
19
|
}
|
|
20
|
-
else {
|
|
21
|
-
|
|
20
|
+
else if (resolved.source === 'direct') {
|
|
21
|
+
await logProgress('Downloaded from remote URL.');
|
|
22
22
|
}
|
|
23
23
|
await logProgress('Extracting frames...');
|
|
24
24
|
let frames = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractFrames.js","sourceRoot":"","sources":["../../src/tools/extractFrames.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"extractFrames.js","sourceRoot":"","sources":["../../src/tools/extractFrames.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAEhD,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EAAE,MAAW,EAAE,EAAE;IACrD,IAAI,OAAO,GAAkB,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,CAAA;QAElE,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAA;QACnD,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAA;QAC1C,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAC1B,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACnC,MAAM,WAAW,CAAC,sCAAsC,CAAC,CAAA;QAC3D,CAAC;aAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxC,MAAM,WAAW,CAAC,6BAA6B,CAAC,CAAA;QAClD,CAAC;QAED,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;QACzC,IAAI,MAAM,GAAa,EAAE,CAAA;QAEzB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;YAC5D,CAAC;YACD,MAAM,GAAG,MAAM,mBAAmB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;QACzD,CAAC;aAAM,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;YACtE,CAAC;YACD,MAAM,GAAG,MAAM,uBAAuB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;QACnE,CAAC;aAAM,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1D,MAAM,IAAI,KAAK,CACb,sEAAsE,CACvE,CAAA;YACH,CAAC;YACD,MAAM,GAAG,MAAM,yBAAyB,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QACpE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjE,CAAC;QAED,IACE,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,IAAI,KAAK,YAAY;YACrB,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EACzB,CAAC;YACD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAiB,EAAE,EAAE,CACnC,cAAc,CAAC,YAAY,EAAE,SAAS,CAAC,CACxC,CACF,CAAA;YACD,MAAM,GAAG,cAAc,CAAA;QACzB,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;QAE1C,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,aAAa,MAAM,CAAC,MAAM,kBAAkB,SAAS,UAAU,IAAI,QAAQ;iBAClF;gBACD,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;oBAC9B,IAAI,EAAE,OAAgB;oBACtB,IAAI,EAAE,oBAAoB,CAAC,WAAW,CAAC;oBACvC,QAAQ,EAAE,WAAW;iBACtB,CAAC,CAAC;aACJ;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC1D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBAC3F;aACF;SACF,CAAA;IACH,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getVideoInfo.d.ts","sourceRoot":"","sources":["../../src/tools/getVideoInfo.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"getVideoInfo.d.ts","sourceRoot":"","sources":["../../src/tools/getVideoInfo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,eAAO,MAAM,gBAAgB,GAAU,QAAQ,GAAG;;;;;;;;;;;;EA+CjD,CAAA"}
|
|
@@ -3,21 +3,22 @@
|
|
|
3
3
|
* Get video metadata
|
|
4
4
|
*/
|
|
5
5
|
import { getVideoMetadata } from '../services/ffmpeg.js';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { cleanupTempDir } from '../utils/tempFiles.js';
|
|
7
|
+
import { resolveVideoInput } from '../utils/videoUtils.js';
|
|
8
8
|
import { logProgress } from '../utils/logger.js';
|
|
9
9
|
export const getVideoInfoTool = async (params) => {
|
|
10
10
|
let tempDir = null;
|
|
11
11
|
try {
|
|
12
12
|
const { videoPath } = params;
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
13
|
+
await logProgress('Resolving video source...');
|
|
14
|
+
const resolved = await resolveVideoInput(videoPath);
|
|
15
|
+
const resolvedPath = resolved.resolvedPath;
|
|
16
|
+
tempDir = resolved.tempDir;
|
|
17
|
+
if (resolved.source === 'platform') {
|
|
18
|
+
await logProgress('Downloaded from platform via yt-dlp.');
|
|
18
19
|
}
|
|
19
|
-
else {
|
|
20
|
-
|
|
20
|
+
else if (resolved.source === 'direct') {
|
|
21
|
+
await logProgress('Downloaded from remote URL.');
|
|
21
22
|
}
|
|
22
23
|
await logProgress('Reading video metadata...');
|
|
23
24
|
const metadata = await getVideoMetadata(resolvedPath);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getVideoInfo.js","sourceRoot":"","sources":["../../src/tools/getVideoInfo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"getVideoInfo.js","sourceRoot":"","sources":["../../src/tools/getVideoInfo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAW,EAAE,EAAE;IACpD,IAAI,OAAO,GAAkB,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAA;QAE5B,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAA;QACnD,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAA;QAC1C,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAC1B,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACnC,MAAM,WAAW,CAAC,sCAAsC,CAAC,CAAA;QAC3D,CAAC;aAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxC,MAAM,WAAW,CAAC,6BAA6B,CAAC,CAAA;QAClD,CAAC;QAED,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAA;QAErD,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;QAE1C,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;wBACE,SAAS;wBACT,GAAG,QAAQ;qBACZ,EACD,IAAI,EACJ,CAAC,CACF;iBACF;aACF;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC1D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBAC5F;aACF;SACF,CAAA;IACH,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -3,17 +3,7 @@
|
|
|
3
3
|
* Find when something specific happens in a video
|
|
4
4
|
*/
|
|
5
5
|
export declare const searchTimestampTool: (params: any) => Promise<{
|
|
6
|
-
content:
|
|
7
|
-
type: "text";
|
|
8
|
-
text: string;
|
|
9
|
-
data?: undefined;
|
|
10
|
-
mimeType?: undefined;
|
|
11
|
-
} | {
|
|
12
|
-
type: "image";
|
|
13
|
-
data: string;
|
|
14
|
-
mimeType: string;
|
|
15
|
-
text?: undefined;
|
|
16
|
-
})[];
|
|
6
|
+
content: any[];
|
|
17
7
|
isError?: undefined;
|
|
18
8
|
} | {
|
|
19
9
|
isError: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"searchTimestamp.d.ts","sourceRoot":"","sources":["../../src/tools/searchTimestamp.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"searchTimestamp.d.ts","sourceRoot":"","sources":["../../src/tools/searchTimestamp.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA6CH,eAAO,MAAM,mBAAmB,GAAU,QAAQ,GAAG;;;;;;;;;EAoMpD,CAAA"}
|