video-context-mcp-server 1.0.0-beta.8 β†’ 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/README.md +754 -171
  2. package/dist/commands/cache.d.ts +43 -0
  3. package/dist/commands/cache.d.ts.map +1 -0
  4. package/dist/commands/cache.js +372 -0
  5. package/dist/commands/cache.js.map +1 -0
  6. package/dist/generated/version.d.ts +2 -0
  7. package/dist/generated/version.d.ts.map +1 -0
  8. package/dist/generated/version.js +3 -0
  9. package/dist/generated/version.js.map +1 -0
  10. package/dist/index.js +88 -77
  11. package/dist/index.js.map +1 -1
  12. package/dist/services/audio/assemblyAiClient.d.ts +21 -0
  13. package/dist/services/audio/assemblyAiClient.d.ts.map +1 -0
  14. package/dist/services/audio/assemblyAiClient.js +109 -0
  15. package/dist/services/audio/assemblyAiClient.js.map +1 -0
  16. package/dist/services/audio/deepgramClient.d.ts +16 -0
  17. package/dist/services/audio/deepgramClient.d.ts.map +1 -0
  18. package/dist/services/audio/deepgramClient.js +117 -0
  19. package/dist/services/audio/deepgramClient.js.map +1 -0
  20. package/dist/services/audio/groqAudioClient.d.ts +16 -0
  21. package/dist/services/audio/groqAudioClient.d.ts.map +1 -0
  22. package/dist/services/audio/groqAudioClient.js +61 -0
  23. package/dist/services/audio/groqAudioClient.js.map +1 -0
  24. package/dist/services/audioRouter.d.ts +54 -0
  25. package/dist/services/audioRouter.d.ts.map +1 -0
  26. package/dist/services/audioRouter.js +103 -0
  27. package/dist/services/audioRouter.js.map +1 -0
  28. package/dist/services/ffmpeg.d.ts +40 -4
  29. package/dist/services/ffmpeg.d.ts.map +1 -1
  30. package/dist/services/ffmpeg.js +176 -52
  31. package/dist/services/ffmpeg.js.map +1 -1
  32. package/dist/services/geminiClient.d.ts +77 -0
  33. package/dist/services/geminiClient.d.ts.map +1 -0
  34. package/dist/services/geminiClient.js +180 -0
  35. package/dist/services/geminiClient.js.map +1 -0
  36. package/dist/services/glmClient.d.ts +20 -2
  37. package/dist/services/glmClient.d.ts.map +1 -1
  38. package/dist/services/glmClient.js +62 -41
  39. package/dist/services/glmClient.js.map +1 -1
  40. package/dist/services/kimiClient.d.ts +2 -1
  41. package/dist/services/kimiClient.d.ts.map +1 -1
  42. package/dist/services/kimiClient.js +4 -1
  43. package/dist/services/kimiClient.js.map +1 -1
  44. package/dist/services/mimoClient.d.ts +107 -0
  45. package/dist/services/mimoClient.d.ts.map +1 -0
  46. package/dist/services/mimoClient.js +251 -0
  47. package/dist/services/mimoClient.js.map +1 -0
  48. package/dist/services/providerRouter.d.ts +88 -7
  49. package/dist/services/providerRouter.d.ts.map +1 -1
  50. package/dist/services/providerRouter.js +197 -18
  51. package/dist/services/providerRouter.js.map +1 -1
  52. package/dist/services/qwenClient.d.ts +59 -0
  53. package/dist/services/qwenClient.d.ts.map +1 -0
  54. package/dist/services/qwenClient.js +199 -0
  55. package/dist/services/qwenClient.js.map +1 -0
  56. package/dist/services/s3Relay.d.ts +61 -0
  57. package/dist/services/s3Relay.d.ts.map +1 -0
  58. package/dist/services/s3Relay.js +154 -0
  59. package/dist/services/s3Relay.js.map +1 -0
  60. package/dist/tools/analyzeVideo.d.ts +5 -13
  61. package/dist/tools/analyzeVideo.d.ts.map +1 -1
  62. package/dist/tools/analyzeVideo.js +199 -62
  63. package/dist/tools/analyzeVideo.js.map +1 -1
  64. package/dist/tools/extractFrames.d.ts +5 -17
  65. package/dist/tools/extractFrames.d.ts.map +1 -1
  66. package/dist/tools/extractFrames.js +146 -32
  67. package/dist/tools/extractFrames.js.map +1 -1
  68. package/dist/tools/getVideoInfo.d.ts +5 -13
  69. package/dist/tools/getVideoInfo.d.ts.map +1 -1
  70. package/dist/tools/getVideoInfo.js +37 -12
  71. package/dist/tools/getVideoInfo.js.map +1 -1
  72. package/dist/tools/orchestrator.d.ts +166 -0
  73. package/dist/tools/orchestrator.d.ts.map +1 -0
  74. package/dist/tools/orchestrator.js +449 -0
  75. package/dist/tools/orchestrator.js.map +1 -0
  76. package/dist/tools/schemas.d.ts +95 -0
  77. package/dist/tools/schemas.d.ts.map +1 -0
  78. package/dist/tools/schemas.js +118 -0
  79. package/dist/tools/schemas.js.map +1 -0
  80. package/dist/tools/searchTimestamp.d.ts +5 -20
  81. package/dist/tools/searchTimestamp.d.ts.map +1 -1
  82. package/dist/tools/searchTimestamp.js +115 -83
  83. package/dist/tools/searchTimestamp.js.map +1 -1
  84. package/dist/tools/summarizeVideo.d.ts +13 -13
  85. package/dist/tools/summarizeVideo.d.ts.map +1 -1
  86. package/dist/tools/summarizeVideo.js +304 -91
  87. package/dist/tools/summarizeVideo.js.map +1 -1
  88. package/dist/tools/transcribeVideo.d.ts +10 -0
  89. package/dist/tools/transcribeVideo.d.ts.map +1 -0
  90. package/dist/tools/transcribeVideo.js +222 -0
  91. package/dist/tools/transcribeVideo.js.map +1 -0
  92. package/dist/types/errors.d.ts +19 -0
  93. package/dist/types/errors.d.ts.map +1 -0
  94. package/dist/types/errors.js +17 -0
  95. package/dist/types/errors.js.map +1 -0
  96. package/dist/types/transcript.d.ts +43 -0
  97. package/dist/types/transcript.d.ts.map +1 -0
  98. package/dist/types/transcript.js +9 -0
  99. package/dist/types/transcript.js.map +1 -0
  100. package/dist/utils/artifactCache.d.ts +292 -0
  101. package/dist/utils/artifactCache.d.ts.map +1 -0
  102. package/dist/utils/artifactCache.js +893 -0
  103. package/dist/utils/artifactCache.js.map +1 -0
  104. package/dist/utils/audioUtils.d.ts +113 -0
  105. package/dist/utils/audioUtils.d.ts.map +1 -0
  106. package/dist/utils/audioUtils.js +322 -0
  107. package/dist/utils/audioUtils.js.map +1 -0
  108. package/dist/utils/base64.d.ts +7 -0
  109. package/dist/utils/base64.d.ts.map +1 -1
  110. package/dist/utils/base64.js +25 -0
  111. package/dist/utils/base64.js.map +1 -1
  112. package/dist/utils/downloadCache.d.ts +102 -0
  113. package/dist/utils/downloadCache.d.ts.map +1 -0
  114. package/dist/utils/downloadCache.js +293 -0
  115. package/dist/utils/downloadCache.js.map +1 -0
  116. package/dist/utils/license.d.ts +110 -0
  117. package/dist/utils/license.d.ts.map +1 -0
  118. package/dist/utils/license.js +358 -0
  119. package/dist/utils/license.js.map +1 -0
  120. package/dist/utils/logger.d.ts +29 -0
  121. package/dist/utils/logger.d.ts.map +1 -0
  122. package/dist/utils/logger.js +49 -0
  123. package/dist/utils/logger.js.map +1 -0
  124. package/dist/utils/tempFiles.d.ts +11 -3
  125. package/dist/utils/tempFiles.d.ts.map +1 -1
  126. package/dist/utils/tempFiles.js +42 -2
  127. package/dist/utils/tempFiles.js.map +1 -1
  128. package/dist/utils/updateCheck.d.ts +54 -0
  129. package/dist/utils/updateCheck.d.ts.map +1 -0
  130. package/dist/utils/updateCheck.js +123 -0
  131. package/dist/utils/updateCheck.js.map +1 -0
  132. package/dist/utils/videoUtils.d.ts +122 -0
  133. package/dist/utils/videoUtils.d.ts.map +1 -0
  134. package/dist/utils/videoUtils.js +618 -0
  135. package/dist/utils/videoUtils.js.map +1 -0
  136. package/dist/utils/ytDlpPath.d.ts +71 -0
  137. package/dist/utils/ytDlpPath.d.ts.map +1 -0
  138. package/dist/utils/ytDlpPath.js +302 -0
  139. package/dist/utils/ytDlpPath.js.map +1 -0
  140. package/package.json +43 -16
package/README.md CHANGED
@@ -1,6 +1,9 @@
1
1
  # Video Context MCP Server
2
2
 
3
- Video Context MCP Server is a Model Context Protocol (MCP) server that gives MCP-compatible coding assistants (such as GitHub Copilot in VS Code, Cursor, and Claude Code) the ability to understand and analyze video content. Supports two AI backends: **Kimi K2.5** (Moonshot AI) and **GLM-4.6V** (Z.AI).
3
+ An MCP server that gives coding assistants (GitHub Copilot, Cursor, Claude Code) the ability to understand and analyze video content.
4
+
5
+ **[Official Documentation](https://www.videocontextmcp.com/)**<br />
6
+ **[Tutorials Playlist](https://www.youtube.com/playlist?list=PL3Big7arQA0Rc1B6WVLmZqXKwDyp1fyme)**
4
7
 
5
8
  ## Features
6
9
 
@@ -9,42 +12,57 @@ Video Context MCP Server is a Model Context Protocol (MCP) server that gives MCP
9
12
  - πŸ–ΌοΈ **Frame Extraction** β€” Extract frames at specific timestamps or intervals
10
13
  - πŸ” **Timestamp Search** β€” Find the exact moment when something happens in a video
11
14
  - πŸ“Š **Video Metadata** β€” Get duration, resolution, fps, codec, and other technical details
12
- - πŸ”„ **Dual Backend Support** β€” Choose between GLM-4.6V (cheap/free) or Kimi K2.5 (broader format support)
15
+ - πŸŽ™οΈ **Audio Transcription** β€” Transcribe speech with paragraph-level timestamps (`[MM:SS]`) or export as SRT/VTT subtitles and JSON using Deepgram, AssemblyAI, Groq/Whisper, or Gemini
16
+ - πŸ”Š **Speaker Diarization** β€” Identify who said what (Deepgram and AssemblyAI)
17
+ - πŸ”Š **Audio-Enhanced Analysis** β€” Auto-transcribes audio and injects transcripts into AI prompts for richer results (GLM/Kimi/Qwen)
18
+ - πŸ”„ **Multi-Provider Support** β€” Choose between GLM-4.6V, Qwen3.6, Kimi K2.5, Gemini, or MiMo-V2 Omni
13
19
  - 🎯 **Smart Video Handling** β€” Extracts keyframes from long videos to reduce token usage
20
+ - ⭐ **Pro tier** β€” Extended frame extraction, multi-platform downloads, higher resolution. [Learn more ↓](#pro)
14
21
 
15
- ## Installation
16
-
17
- ### Quick Start (Recommended for ordinary users)
22
+ ---
18
23
 
19
- #### 1. Prerequisites
24
+ ## Quick Start
20
25
 
21
- - Node.js 18+
22
- - VS Code with GitHub Copilot Chat enabled
26
+ ### Prerequisites
23
27
 
24
- #### 2. Get API keys
28
+ - **Node.js 20+**
29
+ - A supported MCP client (VS Code + GitHub Copilot Chat, Cursor, or Claude Code)
30
+ - At least one video and one audio API key β€” see [API Keys](#api-keys) below
25
31
 
26
- You'll need API keys for one or both backends:
32
+ ### Install
27
33
 
28
- - **Kimi K2.5 (Moonshot AI)**: [Get API Key](https://platform.moonshot.ai)
29
- - **GLM-4.6V (Z.AI)**: [Get API Key](https://z.ai/manage-apikey/apikey-list)
34
+ ```bash
35
+ npm install -g video-context-mcp-server
36
+ ```
30
37
 
31
- #### 3. Install the MCP server
38
+ ### Updating & version check
32
39
 
33
40
  ```bash
34
- npm install -g video-context-mcp-server
41
+ npm ls -g video-context-mcp-server # installed version
42
+ npm outdated -g video-context-mcp-server # check for updates
43
+ npm install -g video-context-mcp-server@latest # update
35
44
  ```
36
45
 
37
- This installs the executable command: `video-context-mcp`.
46
+ ### Configure Your MCP Client
47
+
48
+ <details open>
49
+ <summary><strong>VS Code</strong></summary>
50
+
51
+ You can configure the MCP server globally (for all projects) or at the workspace level.
52
+
53
+ **Global configuration**
38
54
 
39
- > **Tip:** Periodically re-run the above command to get the latest version:
40
- >
41
- > ```bash
42
- > npm install -g video-context-mcp-server@latest
43
- > ```
55
+ Open VS Code β†’ Settings β†’ MCP and add the server configuration.
44
56
 
45
- #### 4. Configure VS Code MCP
57
+ **Workspace-level configuration**
46
58
 
47
- Create (or update) `.vscode/mcp.json` in your project/workspace:
59
+ Create or update `.vscode/mcp.json` in your workspace.
60
+
61
+ > **Important:** This file contains sensitive API keys. **Never commit it to source control.** Ensure it is added to your `.gitignore` file.
62
+
63
+ #### Minimal configuration
64
+
65
+ You only need one video API key to get started. The example below uses Gemini, which has a free tier and requires no credit card:
48
66
 
49
67
  ```json
50
68
  {
@@ -53,19 +71,46 @@ Create (or update) `.vscode/mcp.json` in your project/workspace:
53
71
  "type": "stdio",
54
72
  "command": "video-context-mcp",
55
73
  "env": {
74
+ "GEMINI_API_KEY": "your-gemini-key"
75
+ }
76
+ }
77
+ }
78
+ }
79
+ ```
80
+
81
+ #### Full configuration (all providers)
82
+
83
+ Set all keys to enable the full fallback chain. If one provider is unavailable or rate-limited, the server automatically tries the next:
84
+
85
+ ```json
86
+ {
87
+ "servers": {
88
+ "videoMcp": {
89
+ "type": "stdio",
90
+ "command": "video-context-mcp",
91
+ "env": {
92
+ "GEMINI_API_KEY": "your-gemini-key",
93
+ "Z_AI_API_KEY": "your-zai-key",
94
+ "DASHSCOPE_API_KEY": "your-dashscope-key",
56
95
  "MOONSHOT_API_KEY": "your-moonshot-key",
57
- "Z_AI_API_KEY": "your-zai-key"
96
+ "MIMO_API_KEY": "your-mimo-key",
97
+ "DEEPGRAM_API_KEY": "your-deepgram-key",
98
+ "ASSEMBLYAI_API_KEY": "your-assemblyai-key",
99
+ "GROQ_API_KEY": "your-groq-key"
58
100
  }
59
101
  }
60
102
  }
61
103
  }
62
104
  ```
63
105
 
64
- Open Copilot Chat in VS Code. The MCP server starts automatically when tools are needed.
106
+ Open Copilot Chat β€” the MCP server starts automatically when tools are needed.
107
+
108
+ </details>
65
109
 
66
- ### Configure Cursor MCP
110
+ <details>
111
+ <summary><strong>Cursor</strong></summary>
67
112
 
68
- Add this server to your Cursor MCP configuration (global or project-level):
113
+ You can configure the MCP server globally (for all projects) or at the project level. The configuration format is the same for both:
69
114
 
70
115
  ```json
71
116
  {
@@ -73,65 +118,86 @@ Add this server to your Cursor MCP configuration (global or project-level):
73
118
  "videoMcp": {
74
119
  "command": "video-context-mcp",
75
120
  "env": {
121
+ "GEMINI_API_KEY": "your-gemini-key",
122
+ "Z_AI_API_KEY": "your-zai-key",
123
+ "DASHSCOPE_API_KEY": "your-dashscope-key",
76
124
  "MOONSHOT_API_KEY": "your-moonshot-key",
77
- "Z_AI_API_KEY": "your-zai-key"
125
+ "MIMO_API_KEY": "your-mimo-key",
126
+ "DEEPGRAM_API_KEY": "your-deepgram-key",
127
+ "ASSEMBLYAI_API_KEY": "your-assemblyai-key",
128
+ "GROQ_API_KEY": "your-groq-key"
78
129
  }
79
130
  }
80
131
  }
81
132
  }
82
133
  ```
83
134
 
84
- Notes:
135
+ **Global configuration**
85
136
 
86
- - Run `npm install -g video-context-mcp-server` first if you haven't already.
87
- - If you prefer not to install globally, use `npx -y video-context-mcp-server@latest` as the command (slower startup due to registry check).
88
- - Set one or both API keys depending on which provider you use.
137
+ Open Cursor β†’ Settings β†’ MCP and add the server configuration above.
89
138
 
90
- ### Configure Claude Code MCP
139
+ **Project-level configuration**
91
140
 
92
- Use the Claude CLI to register the MCP server:
141
+ Create a `.cursor/mcp.json` (or `.mcp.json`) in your project root with the configuration above.
93
142
 
94
- ```bash
95
- claude mcp add videoMcp video-context-mcp \
96
- --env MOONSHOT_API_KEY=your-moonshot-key \
97
- --env Z_AI_API_KEY=your-zai-key
98
- ```
143
+ > **Important:** This file contains sensitive API keys. **Never commit it to source control.** Ensure it is added to your `.gitignore` file.
99
144
 
100
- Then verify with:
145
+ </details>
101
146
 
102
- ```bash
103
- claude mcp list
104
- ```
147
+ <details>
148
+ <summary><strong>Claude Code</strong></summary>
105
149
 
106
- If you prefer not to install globally, register via `npx` instead (slower startup due to registry check):
150
+ **Option A β€” CLI (`claude mcp add`)**
107
151
 
108
152
  ```bash
109
- claude mcp add videoMcp npx -y video-context-mcp-server@latest \
153
+ claude mcp add \
154
+ --env GEMINI_API_KEY=your-gemini-key \
155
+ --env Z_AI_API_KEY=your-zai-key \
156
+ --env DASHSCOPE_API_KEY=your-dashscope-key \
110
157
  --env MOONSHOT_API_KEY=your-moonshot-key \
111
- --env Z_AI_API_KEY=your-zai-key
158
+ --env MIMO_API_KEY=your-mimo-key \
159
+ --env DEEPGRAM_API_KEY=your-deepgram-key \
160
+ --env ASSEMBLYAI_API_KEY=your-assemblyai-key \
161
+ --env GROQ_API_KEY=your-groq-key \
162
+ videoMcp -- video-context-mcp
112
163
  ```
113
164
 
114
- ### Troubleshooting Setup
165
+ Verify with `claude mcp list`.
166
+
167
+ **Option B β€” project-level `.mcp.json`**
168
+
169
+ Create `.mcp.json` in your project root:
115
170
 
116
- - **`video-context-mcp: command not found`**
117
- - Make sure Node.js is installed and available in your shell (`node -v`, `npm -v`).
118
- - If installed globally, re-run: `npm install -g video-context-mcp-server`.
119
- - If global binaries are not on PATH, use `npx -y video-context-mcp-server@latest` instead of `video-context-mcp`.
171
+ > **Important:** This file contains sensitive API keys. **Never commit it to source control.** Ensure it is added to your `.gitignore` file.
172
+
173
+ ```json
174
+ {
175
+ "mcpServers": {
176
+ "videoMcp": {
177
+ "command": "video-context-mcp",
178
+ "env": {
179
+ "GEMINI_API_KEY": "your-gemini-key",
180
+ "Z_AI_API_KEY": "your-zai-key",
181
+ "DASHSCOPE_API_KEY": "your-dashscope-key",
182
+ "MOONSHOT_API_KEY": "your-moonshot-key",
183
+ "MIMO_API_KEY": "your-mimo-key",
184
+ "DEEPGRAM_API_KEY": "your-deepgram-key",
185
+ "ASSEMBLYAI_API_KEY": "your-assemblyai-key",
186
+ "GROQ_API_KEY": "your-groq-key"
187
+ }
188
+ }
189
+ }
190
+ }
191
+ ```
120
192
 
121
- - **MCP server not appearing in client**
122
- - Restart the client app after changing MCP configuration.
123
- - Validate JSON syntax in your MCP config file.
124
- - For Claude Code, verify registration with `claude mcp list`.
193
+ </details>
125
194
 
126
- - **Missing API key errors**
127
- - Set `Z_AI_API_KEY` for GLM usage and `MOONSHOT_API_KEY` for Kimi usage.
128
- - You can set only one key if you only use one provider.
129
- - For local files, if `provider=glm` is requested but `Z_AI_API_KEY` is missing, the server automatically falls back to Kimi when `MOONSHOT_API_KEY` is available.
130
- - For remote `http(s)` video URLs, use `provider=glm` (Kimi requires local file input).
195
+ <details>
196
+ <summary><strong>Run via npx (no global install)</strong></summary>
131
197
 
132
- ### Alternative: Run via npx without global install
198
+ Use `npx -y video-context-mcp-server@latest` as the command instead of `video-context-mcp`. This adds a startup delay (npm registry check) but self-updates automatically.
133
199
 
134
- If you prefer not to install globally, you can use `npx` instead. Note this adds a startup delay due to the npm registry check on each run:
200
+ **VS Code example:**
135
201
 
136
202
  ```json
137
203
  {
@@ -141,137 +207,516 @@ If you prefer not to install globally, you can use `npx` instead. Note this adds
141
207
  "command": "npx",
142
208
  "args": ["-y", "video-context-mcp-server@latest"],
143
209
  "env": {
210
+ "GEMINI_API_KEY": "your-gemini-key",
211
+ "Z_AI_API_KEY": "your-zai-key",
212
+ "DASHSCOPE_API_KEY": "your-dashscope-key",
144
213
  "MOONSHOT_API_KEY": "your-moonshot-key",
145
- "Z_AI_API_KEY": "your-zai-key"
214
+ "MIMO_API_KEY": "your-mimo-key",
215
+ "DEEPGRAM_API_KEY": "your-deepgram-key",
216
+ "ASSEMBLYAI_API_KEY": "your-assemblyai-key",
217
+ "GROQ_API_KEY": "your-groq-key"
146
218
  }
147
219
  }
148
220
  }
149
221
  }
150
222
  ```
151
223
 
152
- The `@latest` tag ensures you always get the newest published version, at the cost of a network round-trip on every startup β€” this also means the `npx` approach self-updates automatically and requires no manual update step.
224
+ **Claude Code:**
225
+
226
+ ```bash
227
+ claude mcp add \
228
+ --env GEMINI_API_KEY=your-gemini-key \
229
+ --env Z_AI_API_KEY=your-zai-key \
230
+ --env DASHSCOPE_API_KEY=your-dashscope-key \
231
+ --env MOONSHOT_API_KEY=your-moonshot-key \
232
+ --env MIMO_API_KEY=your-mimo-key \
233
+ --env DEEPGRAM_API_KEY=your-deepgram-key \
234
+ --env ASSEMBLYAI_API_KEY=your-assemblyai-key \
235
+ --env GROQ_API_KEY=your-groq-key \
236
+ videoMcp -- npx -y video-context-mcp-server@latest
237
+ ```
238
+
239
+ </details>
153
240
 
154
- ### For contributors: install from source
241
+ > **Tip:** Set keys for all providers you can access β€” this maximises the fallback chain. If the default provider is unavailable or rate-limited, the server automatically retries with the next available one. At minimum you need one video key and one audio key, but more is better.
155
242
 
156
- Clone this repository, then:
243
+ ---
244
+
245
+ ## Available Tools
246
+
247
+ | Tool | Description | Key Parameters |
248
+ | ------------------ | ------------------------------------ | ------------------------------------------------------------------------------------------- |
249
+ | `analyze_video` | Ask questions about video content | `videoPath`, `question`, `provider?` |
250
+ | `summarize_video` | Generate a structured video summary | `videoPath`, `provider?` |
251
+ | `extract_frames` | Extract frames from a video | `videoPath`, `mode`, `count/intervalSec/timestamps/sceneThreshold`, `maxImages?`, `offset?` |
252
+ | `search_timestamp` | Find when something specific happens | `videoPath`, `query`, `provider?` |
253
+ | `get_video_info` | Get video metadata | `videoPath` |
254
+ | `transcribe_video` | Transcribe audio/speech from a video | `videoPath`, `provider?`, `language?`, `diarize?`, `translate?`, `outputFormat?` |
255
+
256
+ All tools accept local file paths, `file://` URIs, and remote `http(s)` URLs. Remote videos are downloaded automatically.
257
+
258
+ ### Usage Examples
259
+
260
+ **Local files**
261
+
262
+ > "Analyze the video at `./demo.mp4` and tell me what happens in it"
263
+
264
+ > "Summarize `./long-video.mp4`"
265
+
266
+ > "Extract 5 evenly-spaced frames from `./recording.mp4`"
267
+
268
+ > "Extract frames at scene changes from `./video.mp4`" (set `mode: scene`)
269
+
270
+ > "At what timestamp does the person wave in `./clip.mp4`?"
271
+
272
+ > "Transcribe `./interview.mp4` with speaker diarization using AssemblyAI"
273
+
274
+ > "Transcribe this Spanish video and translate it to English: `./video.mp4`"
275
+
276
+ > "Generate SRT subtitles for `./interview.mp4`" (set `outputFormat: srt`)
277
+
278
+ > "Export a WebVTT subtitle file from `./talk.mp4`" (set `outputFormat: vtt`)
279
+
280
+ **Remote video files** (direct URLs β€” works for all users)
281
+
282
+ > "Analyze this video: `https://example.com/product-demo.mp4`"
283
+
284
+ > "Get the metadata for `https://cdn.example.com/sample.webm`"
285
+
286
+ **Platform videos** (YouTube, Vimeo, TikTok, Bilibili, etc. β€” requires [Pro](#pro) for non-YouTube)
287
+
288
+ > "Summarize this YouTube video: `https://www.youtube.com/watch?v=dQw4w9WgXcQ`"
289
+
290
+ > "At what point does the speaker mention pricing in `https://www.youtube.com/watch?v=abc123`?"
291
+
292
+ > "Transcribe the audio from `https://vimeo.com/123456789`"
293
+
294
+ ---
295
+
296
+ ## API Keys
297
+
298
+ ### Video Providers
299
+
300
+ Set all keys to get the full fallback chain. The server will try Gemini first, then GLM, then Qwen, then Kimi, then MiMo, so having all keys ensures it never gets stuck without a working provider:
301
+
302
+ | Provider | Key | Link |
303
+ | ----------------------------------------------- | ------------------- | --------------------------------------------------------------------------------------------- |
304
+ | **Gemini 3 Flash Preview** (default, free-tier) | `GEMINI_API_KEY` | [Get key](https://aistudio.google.com/app/apikey) |
305
+ | **GLM-4.6V** (free-tier) | `Z_AI_API_KEY` | [Get key](https://z.ai/manage-apikey/apikey-list) |
306
+ | **Qwen3.6** (FREE β€” promotional) | `DASHSCOPE_API_KEY` | [Get key](https://modelstudio.console.alibabacloud.com/ap-southeast-1?tab=dashboard#/api-key) |
307
+ | **Kimi K2.5** (paid) | `MOONSHOT_API_KEY` | [Get key](https://platform.moonshot.ai) |
308
+ | **MiMo-V2 Omni** (paid) | `MIMO_API_KEY` | [Get key](https://platform.xiaomimimo.com/#/console/api-keys) |
309
+
310
+ When a provider's API key is missing or its API call fails at runtime, tools automatically fall back through the remaining providers in priority order, starting from the configured default:
311
+
312
+ - Gemini default (standard): **Gemini β†’ GLM β†’ Qwen β†’ Kimi β†’ MiMo**
313
+ - GLM default: **GLM β†’ Gemini β†’ Qwen β†’ Kimi β†’ MiMo**
314
+ - Qwen default: **Qwen β†’ Gemini β†’ GLM β†’ Kimi β†’ MiMo**
315
+ - Kimi default: **Kimi β†’ Gemini β†’ GLM β†’ Qwen β†’ MiMo**
316
+ - MiMo default: **MiMo β†’ Gemini β†’ GLM β†’ Qwen β†’ Kimi**
317
+
318
+ ### Audio Providers
319
+
320
+ Similarly, set all audio keys so transcription always has a fallback provider available:
321
+
322
+ | Provider | Key | Link |
323
+ | ----------------------------------------- | -------------------- | ----------------------------------------------- |
324
+ | **Deepgram** (default, $200 free credits) | `DEEPGRAM_API_KEY` | [Get key](https://console.deepgram.com/) |
325
+ | **AssemblyAI** ($50 free credits) | `ASSEMBLYAI_API_KEY` | [Get key](https://www.assemblyai.com/dashboard) |
326
+ | **Groq/Whisper** (free-tier) | `GROQ_API_KEY` | [Get key](https://console.groq.com/) |
327
+ | **Gemini** (free-tier) | `GEMINI_API_KEY` | Reuses the same key as the video provider |
328
+
329
+ When an audio key is missing or an audio API call fails at runtime, tools automatically fall back through the remaining providers in priority order, starting from the configured default (e.g. with Deepgram default: **Deepgram β†’ AssemblyAI β†’ Groq β†’ Gemini**).
330
+
331
+ ---
332
+
333
+ ## Provider Comparison
334
+
335
+ ### Video Providers
336
+
337
+ | Feature | Gemini 3 Flash Preview (default) | GLM-4.6V | Qwen3.6 | Kimi K2.5 | MiMo-V2 Omni |
338
+ | -------------- | ---------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------ | ---------------------------------------------- | ------------------------------------------------------ |
339
+ | Price | Free tier available | Free tier available (GLM-4.6V-Flash) | **FREE** (promotional launch period) | $0.60 input / $3.00 output per 1M tokens | $0.40 input / $2.00 output per 1M tokens |
340
+ | Video formats | mp4, mpeg, mov, avi, flv, mpg, webm, wmv, 3gpp | mp4, avi, mov, wmv, webm, m4v | mp4, avi, mov, wmv, webm, m4v | mp4, mpeg, mov, avi, flv, mpg, webm, wmv, 3gpp | mp4, mov, avi, wmv |
341
+ | Context window | 1M tokens | 128K tokens | 1M tokens | 256K tokens | 256K tokens |
342
+ | Max file size | 2 GB | ~12 MB base64 / frames fallback / **unlimited w/ S3↓** | ~10 MB base64 / frames fallback / **unlimited w/ S3↓** | 100 MB | ~10 MB base64 / frames fallback / **unlimited w/ S3↓** |
343
+ | Best for | **Default** β€” free, no card required | Free, no card required | **FREE during promo** β€” SOTA agentic coding | Paid β€” broadest format support | Paid β€” thinking mode; multimodal |
344
+
345
+ **Gemini 3 Flash Preview** is the default β€” it offers a free tier with no credit card required, 1M context window, and 2 GB file support. **GLM-4.6V** is the second fallback β€” also free with no card required. **Qwen3.6** is currently **FREE** during its promotional launch period (released April 2, 2026) β€” standard pricing will be $0.50 input / $3.00 output per 1M tokens. **Kimi K2.5** is a paid provider with the broadest video format support. **MiMo-V2 Omni** is Xiaomi's multimodal model with thinking mode support ($0.40 input / $2.00 output per 1M tokens).
346
+
347
+ Set `VIDEO_MCP_DEFAULT_PROVIDER=gemini`, `glm`, `qwen`, `kimi`, or `mimo` to change the default provider used when a tool call does not pass `provider`. If a tool call includes `provider`, that per-call value takes precedence.
348
+
349
+ > **ℹ️ Large local files with GLM, Qwen, or MiMo:** All three providers have a 10–12 MB base64 limit for local files. When a file exceeds this limit, the server first tries to fall back to an upload-capable provider (Gemini or Kimi) if one is available in the fallback chain. Frame-based analysis (evenly-spaced keyframes sent as images) is used only as a **last resort** when no upload-capable provider is available, or when all upload-capable providers fail at runtime β€” no configuration needed. For the highest quality with large local videos, set up the optional **S3 relay** (below) β€” GLM, Qwen, and MiMo will receive a presigned URL to the full video, bypassing the limit entirely and taking priority over both fallbacks.
350
+
351
+ <details open>
352
+ <summary><strong>Automatic S3 relay: bypass the 10 MB local file limit with GLM, Qwen, and MiMo</strong></summary>
353
+
354
+ **GLM-4.6V**, **Qwen3.6**, and **MiMo-V2 Omni** all accept direct video URLs, but base64-encoding a local file caps out at **10–12 MB**. Above that limit, the server first tries to fall back to an upload-capable provider (Gemini or Kimi) if one is available, then falls back to **frame-based analysis** as a last resort. For the best results on large local videos, set `AWS_S3_BUCKET` β€” the server uploads the full video to S3 and passes a presigned URL to GLM, Qwen, and MiMo, bypassing the base64 limit entirely and taking priority over both fallbacks. No manual upload step needed.
355
+
356
+ #### Why S3 works
357
+
358
+ GLM, Qwen, and MiMo require the serving endpoint to provide `Content-Length` and `Content-Type` headers alongside the video. AWS S3 presigned URLs include both headers automatically.
359
+
360
+ ---
361
+
362
+ #### Prerequisites
363
+
364
+ Before setting up the S3 relay, you'll need an AWS account and access credentials.
365
+
366
+ ##### 1. Create an AWS account
367
+
368
+ 1. Go to [aws.amazon.com](https://aws.amazon.com/) and click **Create an AWS Account**.
369
+ 2. Enter your email address, a password, and an AWS account name.
370
+ 3. Choose the **Basic Support β€” Free** plan (sufficient for S3 relay usage).
371
+ 4. Fill in your contact and billing information. A valid credit or debit card is required, but S3 usage within the free tier costs nothing.
372
+ 5. Verify your identity via phone call or SMS.
373
+ 6. Once confirmed, sign in to the [AWS Management Console](https://console.aws.amazon.com/).
374
+
375
+ > New AWS accounts include a 12-month Free Tier with 5 GB of S3 storage and 20,000 GET requests per month β€” more than enough for typical video analysis workflows.
376
+
377
+ ##### 2. Get your AWS Access Key ID and Secret Access Key
378
+
379
+ The S3 relay needs programmatic access to your S3 bucket. You'll create an IAM user with limited permissions:
380
+
381
+ 1. In the AWS Console, search for **IAM** in the top search bar and open the IAM dashboard.
382
+ 2. Click **Users** in the left sidebar, then **Create user**.
383
+ 3. Enter a user name (e.g., `video-mcp-s3`) and click **Next**.
384
+ 4. Under **Permissions options**, select **Attach policies directly**.
385
+ 5. Click **Create policy** β€” this opens a new tab:
386
+ - Switch to the **JSON** tab and paste the following minimum-permission policy:
387
+ ```json
388
+ {
389
+ "Version": "2012-10-17",
390
+ "Statement": [
391
+ {
392
+ "Effect": "Allow",
393
+ "Action": [
394
+ "s3:PutObject",
395
+ "s3:GetObject",
396
+ "s3:DeleteObject",
397
+ "s3:ListBucket"
398
+ ],
399
+ "Resource": [
400
+ "arn:aws:s3:::your-globally-unique-bucket-name",
401
+ "arn:aws:s3:::your-globally-unique-bucket-name/*"
402
+ ]
403
+ }
404
+ ]
405
+ }
406
+ ```
407
+
408
+ - Replace `your-globally-unique-bucket-name` with your actual globally unique bucket name (you'll create it in the next step).
409
+ - Click **Next**, give the policy a name like `VideoMcpS3Access`, then **Create policy**.
410
+
411
+ 6. Go back to the user creation tab, click the refresh icon, search for `VideoMcpS3Access`, select it, and click **Next** β†’ **Create user**.
412
+ 7. Open the newly created user, go to the **Security credentials** tab.
413
+ 8. Under **Access keys**, click **Create access key**.
414
+ 9. On **Access key best practices & alternatives**, choose **Other** or the closest equivalent programmatic/local-code option shown in your console, then click **Next**.
415
+ 10. Optionally add a description tag, then click **Create access key**.
416
+ 11. Copy and save the **Access Key ID** and **Secret Access Key** β€” you won't be able to see the secret key again after closing this dialog.
417
+
418
+ > **Never commit your Secret Access Key to version control or share it publicly.** Only add it to your local MCP configuration or AWS credentials file.
419
+
420
+ ##### 3. (Optional) Install and configure the AWS CLI
421
+
422
+ The AWS CLI is only needed if you want to create buckets from the terminal or use the `~/.aws/credentials` method instead of environment variables. If you plan to add credentials directly to your MCP `env` block, you can skip this step.
423
+
424
+ **Install the AWS CLI**
425
+
426
+ - **Windows:** Download the installer from [aws.amazon.com/cli](https://aws.amazon.com/cli/) or run:
427
+ ```bash
428
+ winget install Amazon.AWSCLI
429
+ ```
430
+ - **macOS:**
431
+ ```bash
432
+ curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg"
433
+ sudo installer -pkg AWSCLIV2.pkg -target /
434
+ ```
435
+ - **Linux:**
436
+ ```bash
437
+ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
438
+ unzip awscliv2.zip
439
+ sudo ./aws/install
440
+ ```
441
+
442
+ **Configure your credentials**
443
+
444
+ Run the following command and paste your Access Key ID and Secret Access Key when prompted:
157
445
 
158
446
  ```bash
159
- npm install
160
- npm run build
447
+ aws configure
161
448
  ```
162
449
 
163
- Then use this `.vscode/mcp.json` server command:
450
+ You'll be asked for:
451
+
452
+ - **AWS Access Key ID** β€” paste the key you saved earlier
453
+ - **AWS Secret Access Key** β€” paste the secret key you saved earlier
454
+ - **Default region name** β€” enter your preferred region (e.g., `us-east-1`)
455
+ - **Default output format** β€” press Enter for `json`
456
+
457
+ This stores your credentials in `~/.aws/credentials` and `~/.aws/config`, which the MCP server reads automatically.
458
+
459
+ ---
460
+
461
+ #### One-time setup
462
+
463
+ ##### 1. Create an S3 bucket
464
+
465
+ If you installed the AWS CLI:
466
+
467
+ ```bash
468
+ aws s3 mb s3://your-globally-unique-bucket-name
469
+ ```
470
+
471
+ Or create it manually in the [S3 Console](https://s3.console.aws.amazon.com/):
472
+
473
+ 1. Open the S3 dashboard and click **Create bucket**.
474
+ 2. Enter a globally unique bucket name (e.g., `your-globally-unique-bucket-name`).
475
+ 3. Choose the AWS Region you want to use. This should match `AWS_REGION` in your MCP config or AWS CLI profile.
476
+ 4. Leave **Block all public access** enabled. The bucket does not need to be public β€” the server uses presigned URLs.
477
+ 5. Keep the default **Object Ownership** setting (`ACLs disabled` / `Bucket owner enforced`).
478
+ 6. Leave the remaining settings at their defaults, then click **Create bucket**.
479
+
480
+ > You do not need to add a bucket policy or make objects public. A private bucket works fine because the MCP server generates time-limited presigned URLs for each uploaded video.
481
+
482
+ ##### 2. Add `AWS_S3_BUCKET` to your MCP config
483
+
484
+ **VS Code (`.vscode/mcp.json`)**
164
485
 
165
486
  ```json
166
487
  {
167
488
  "servers": {
168
489
  "videoMcp": {
169
490
  "type": "stdio",
170
- "command": "node",
171
- "args": ["${workspaceFolder}/dist/index.js"],
491
+ "command": "video-context-mcp",
172
492
  "env": {
173
- "MOONSHOT_API_KEY": "your-moonshot-key",
174
- "Z_AI_API_KEY": "your-zai-key"
493
+ "AWS_S3_BUCKET": "your-globally-unique-bucket-name",
494
+ "GEMINI_API_KEY": "your-gemini-key"
175
495
  }
176
496
  }
177
497
  }
178
498
  }
179
499
  ```
180
500
 
181
- ### Debugging Behavior in VS Code
501
+ ---
502
+
503
+ #### AWS Credential Resolution
504
+
505
+ The server resolves AWS credentials in this order β€” you only need to configure one:
506
+
507
+ 1. **Environment variables** β€” add directly to your MCP `env` block (no AWS CLI needed):
508
+ ```jsonc
509
+ "AWS_S3_BUCKET": "your-globally-unique-bucket-name",
510
+ "AWS_ACCESS_KEY_ID": "AKIA...",
511
+ "AWS_SECRET_ACCESS_KEY": "your-secret-key",
512
+ "AWS_REGION": "us-east-1"
513
+ ```
514
+ 2. **`~/.aws/credentials`** β€” if the AWS CLI is configured, credentials are picked up automatically. Only `AWS_S3_BUCKET` is needed in your MCP config:
515
+ ```jsonc
516
+ "AWS_S3_BUCKET": "your-globally-unique-bucket-name"
517
+ ```
518
+ 3. **IAM instance role / ECS task role** β€” for AWS-hosted environments.
519
+
520
+ ---
521
+
522
+ #### How it works at runtime
182
523
 
183
- When your `.vscode/mcp.json` includes a `dev` block such as:
524
+ Every time you analyze a local video (or a platform download like YouTube) with GLM, Qwen, or MiMo:
525
+
526
+ 1. The server detects the file is too large for base64 encoding.
527
+ 2. The file is uploaded to `s3://your-globally-unique-bucket-name/video-mcp-relay/<uuid>.<ext>`.
528
+ 3. A presigned URL (valid for 1 hour) is passed to the AI provider.
529
+ 4. The provider downloads the video directly from S3.
530
+ 5. The object is kept in the bucket for reuse within the same session.
531
+
532
+ **Cleanup:** Relayed S3 objects are deleted automatically when the MCP server session ends. Orphaned objects from crashed sessions are swept at next startup.
533
+
534
+ To keep objects in the bucket for reuse across sessions (useful for large files you analyze repeatedly):
184
535
 
185
536
  ```jsonc
186
- "dev": {
187
- "watch": "src/**/*.ts",
188
- "debug": { "type": "node" }
189
- }
537
+ "AWS_S3_RELAY_CLEANUP": "false"
190
538
  ```
191
539
 
192
- you may see frequent logs in the **Output** panel under `MCP: videoMcp`.
540
+ **Cost**: AWS S3 free tier covers 5 GB storage + 20K GET requests/month for 12 months. After the free tier, storage costs roughly $0.023/GB/month β€” negligible for most use cases.
541
+
542
+ ---
193
543
 
194
- This is expected in development mode:
544
+ #### Manual presigned URLs (alternative)
195
545
 
196
- - `watch` restarts/reloads the MCP server when TypeScript files change
197
- - `debug` enables Node debug integration
198
- - MCP protocol payloads (tool schemas, discovery events, lifecycle messages) are printed in that output channel
546
+ You can also pass a presigned URL directly to any tool without configuring the relay:
199
547
 
200
- If you want less noise, remove either:
548
+ ```bash
549
+ aws s3 cp my-video.mp4 s3://your-globally-unique-bucket-name/my-video.mp4
550
+ aws s3 presign s3://your-globally-unique-bucket-name/my-video.mp4 --expires-in 3600
551
+ # β†’ https://your-globally-unique-bucket-name.s3.amazonaws.com/my-video.mp4?X-Amz-...
552
+ ```
201
553
 
202
- - `dev.debug` (keeps auto-watch, disables debug integration), or
203
- - the full `dev` block (disables watch + debug behavior)
554
+ Then pass the URL directly to `analyze_video`, `summarize_video`, or any other tool.
204
555
 
205
- ## Available Tools
556
+ </details>
557
+
558
+ ### Audio Providers
206
559
 
207
- | Tool | Description | Parameters |
208
- | ------------------ | ------------------------------------ | --------------------------------------------------- |
209
- | `analyze_video` | Ask questions about video content | `videoPath`, `question`, `provider?` |
210
- | `summarize_video` | Generate a structured video summary | `videoPath`, `provider?` |
211
- | `extract_frames` | Extract frames from a video | `videoPath`, `mode`, `count/intervalSec/timestamps` |
212
- | `search_timestamp` | Find when something specific happens | `videoPath`, `query`, `provider?` |
213
- | `get_video_info` | Get video metadata | `videoPath` |
560
+ | Feature | Deepgram (default) | AssemblyAI | Groq/Whisper | Gemini |
561
+ | ------------------- | ---------------------------- | ------------------------ | ----------------------- | -------------------------- |
562
+ | Price | Paid ($200 free credits) | Paid ($50 free credits) | Free tier available | Free tier available |
563
+ | Speaker diarization | Yes | Yes | No | No |
564
+ | English Translation | No | No | Yes | Yes |
565
+ | Best for | **Default** (fast, accurate) | High-quality diarization | Free/cost-conscious use | Users already using Gemini |
214
566
 
215
- ### Path and Provider Constraints
567
+ > **Note:** "Translation" here means speech-to-English conversion (output English regardless of the spoken language). It is distinct from multilingual transcription β€” Deepgram and AssemblyAI both support transcribing dozens of languages natively, but they always output in the source language. Use Groq or Gemini when you need an English transcript of non-English audio.
216
568
 
217
- - `analyze_video` and `summarize_video` support local files and `http(s)` URLs.
218
- - For remote `http(s)` URLs, only `provider=glm` is supported. `provider=kimi` requires a local file.
219
- - `search_timestamp`, `extract_frames`, and `get_video_info` currently support local files only.
220
- - For local inputs, `analyze_video`, `summarize_video`, and `search_timestamp` accept normal paths or `file://` URIs.
221
- - `extract_frames` and `get_video_info` currently expect local filesystem path strings.
569
+ Set `AUDIO_MCP_DEFAULT_PROVIDER` to change the default.
222
570
 
223
- ## Usage Examples
571
+ ---
224
572
 
225
- ### Analyze Video
573
+ ## Environment Variables
226
574
 
227
- Ask Copilot Chat:
575
+ ### Core
228
576
 
229
- > "Analyze the video at `./demo.mp4` and tell me what happens in it"
577
+ | Variable | Description | Default |
578
+ | ---------------------------- | --------------------------------------------------------------------------------------------------------------------- | -------- |
579
+ | `Z_AI_API_KEY` | Z.AI API key for GLM-4.6V | β€” |
580
+ | `DASHSCOPE_API_KEY` | Alibaba Cloud API key for Qwen3.6 | β€” |
581
+ | `MOONSHOT_API_KEY` | Moonshot AI API key for Kimi K2.5 | β€” |
582
+ | `GEMINI_API_KEY` | Google API key for Gemini | β€” |
583
+ | `MIMO_API_KEY` | Xiaomi MiMo API key for MiMo-V2 Omni | β€” |
584
+ | `VIDEO_MCP_DEFAULT_PROVIDER` | Default video provider used when a tool call does not pass `provider`; a per-call `provider` argument can override it | `gemini` |
230
585
 
231
- ### Summarize Video
586
+ ### S3 Relay
232
587
 
233
- > "Summarize the video at `./long-video.mp4`"
588
+ | Variable | Description | Default |
589
+ | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
590
+ | `AWS_S3_BUCKET` | S3 bucket name for automatic video relay. When set, local videos (and platform downloads) are uploaded to S3 and GLM/Qwen receive a presigned URL β€” bypassing the 10–12 MB base64 limit. | β€” |
591
+ | `AWS_ACCESS_KEY_ID` | AWS access key ID. Required if you are not using `~/.aws/credentials` or an IAM role. | β€” |
592
+ | `AWS_SECRET_ACCESS_KEY` | AWS secret access key. Required alongside `AWS_ACCESS_KEY_ID`. | β€” |
593
+ | `AWS_REGION` | AWS region for the S3 bucket (e.g. `us-east-1`). Required if not set in `~/.aws/config`. | β€” |
594
+ | `AWS_S3_RELAY_CLEANUP` | Set to `false` to keep relayed S3 objects in the bucket for reuse across sessions. Default: relayed objects are deleted when the MCP server session ends **and** orphaned objects are swept at startup. | `true` |
234
595
 
235
- ### Extract Frames
596
+ ### Audio
236
597
 
237
- > "Extract 5 evenly-spaced frames from `./video.mp4`"
598
+ | Variable | Description | Default |
599
+ | ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |
600
+ | `DEEPGRAM_API_KEY` | Deepgram API key | β€” |
601
+ | `ASSEMBLYAI_API_KEY` | AssemblyAI API key | β€” |
602
+ | `GROQ_API_KEY` | Groq API key for Whisper transcription | β€” |
603
+ | `AUDIO_MCP_DEFAULT_PROVIDER` | Default audio provider: `deepgram`, `assemblyai`, `groq`, or `gemini`. Falls back in that order when the selected key is missing. | `deepgram` |
604
+ | `AUDIO_ENHANCE_VIDEO_ANALYSIS` | Inject audio transcripts into `analyze_video`/`summarize_video` prompts (GLM/Kimi/Qwen/MiMo). `auto` = transcribe when audio track detected; `true` = always; `false` = disabled. | `auto` |
238
605
 
239
- > "Extract a frame at timestamp 30 seconds from `./video.mp4`"
606
+ ### Caching
240
607
 
241
- ### Search Timestamp
608
+ Downloaded videos, extracted frames, audio tracks, and transcripts are cached together in a persistent per-video directory. Subsequent tool calls that reference the same video reuse cached artifacts instead of re-running ffmpeg, re-downloading, or re-calling audio provider APIs.
242
609
 
243
- > "In `./video.mp4`, at what timestamp does the person wave?"
610
+ | Variable | Description | Default |
611
+ | ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ----------------- |
612
+ | `VIDEO_MCP_CACHE_TTL_MINUTES` | How long video artifacts (video file, frames, audio, transcripts) are cached across tool calls (minutes). Set `0` to disable. | `43200` (30 days) |
613
+ | `VIDEO_MCP_CACHE_MAX_ENTRIES` | Max entries in the artifact cache. LRU eviction at the video level. Set `0` for unbounded. | `100` |
614
+ | `VIDEO_MCP_CACHE_MAX_MB` | Max total disk size for the cache (megabytes). LRU eviction when the limit is reached. Set `0` to disable. | `5120` (5 GB) |
244
615
 
245
- ### Get Video Info
616
+ #### Cache Storage Location
246
617
 
247
- > "Get the video info for `./video.mp4`"
618
+ The cache is stored in a `video-mcp-cache` folder within your system's temporary directory:
248
619
 
249
- ## Additional Guides
620
+ - **Windows:** `%TEMP%\video-mcp-cache`
621
+ (e.g., `C:\Users\<you>\AppData\Local\Temp\video-mcp-cache`)
622
+ - **macOS:** `$TMPDIR/video-mcp-cache`
623
+ (e.g., `/var/folders/xx/yyyy/T/video-mcp-cache`)
624
+ - **Linux:** `/tmp/video-mcp-cache`
250
625
 
251
- - [Screen Recording for Small File Sizes (Windows)](docs/screen-recording-small-files.md)
626
+ #### Automatic Cleanup
252
627
 
253
- ## Backend Comparison
628
+ The server automatically manages the cache by:
254
629
 
255
- | Feature | GLM-4.6V | Kimi K2.5 |
256
- | -------------- | ----------------------------- | ---------------------------------------------- |
257
- | Video formats | mp4, avi, mov, wmv, webm, m4v | mp4, mpeg, mov, avi, flv, mpg, webm, wmv, 3gpp |
258
- | Price | $0.30 input / $0.90 output | $0.60 input / $3.00 output |
259
- | Free tier | Yes (GLM-4.6V-Flash) | No |
260
- | Context window | 128K | 256K |
261
- | Best for | Cost efficiency, free tier | Broader format support |
630
+ 1. **Startup Sweep:** Removing any cache entries older than the TTL at server startup.
631
+ 2. **LRU Eviction:** Evicting the least-recently-used video entry (all its artifacts together) when the `VIDEO_MCP_CACHE_MAX_ENTRIES` limit is reached.
632
+ 3. **Size Eviction:** Evicting the least-recently-used video entry when the total cache size exceeds `VIDEO_MCP_CACHE_MAX_MB` (default 5 GB).
262
633
 
263
- **GLM-4.6V is the default backend** because it offers a free tier (GLM-4.6V-Flash) β€” meaning you can get started at no cost. If you have spare API credits or budget to spend, Kimi K2.5 is worth trying for its broader format support and larger context window. Set `VIDEO_MCP_DEFAULT_PROVIDER=kimi` to switch the default. This env default is used when a tool call omits the `provider` parameter.
634
+ #### Cache Management CLI
264
635
 
265
- ## Environment Variables
636
+ `video-context-mcp` (or the short alias `vmcp`) ships with built-in cache management commands. Run them from any terminal β€” they complete immediately and do **not** start the MCP server.
637
+
638
+ ```bash
639
+ # Show cache location, size, entry count, TTL, and per-entry breakdown
640
+ vmcp cache status
641
+
642
+ # Print the cache directory path (useful for scripting)
643
+ vmcp cache path
644
+ cd "$(vmcp cache path)"
645
+
646
+ # Copy the entire cache tree to ./video-mcp-cache (or a custom path)
647
+ vmcp cache copy
648
+
649
+ # macOS / Linux β€” absolute path
650
+ vmcp cache copy --dest=/Users/alice/projects/cache-backup
651
+ vmcp cache copy --dest=/tmp/cache-backup
652
+
653
+ # Windows β€” absolute path (both \ and / are supported)
654
+ vmcp cache copy --dest=C:\Users\alice\projects\cache-backup
655
+ vmcp cache copy --dest=C:/Users/alice/projects/cache-backup
656
+
657
+ # Relative path (resolved from CWD β€” works on all platforms)
658
+ vmcp cache copy --dest=./backup
659
+ vmcp cache copy --dest=../sibling/cache
660
+
661
+ # Paths with spaces β€” wrap the value in quotes
662
+ vmcp cache copy --dest="/Users/alice/My Projects/cache-backup"
663
+ vmcp cache copy --dest="C:\Users\alice\My Documents\cache-backup"
664
+
665
+ # Remove only entries older than the TTL (defaults to 30 days if not set)
666
+ vmcp cache clear:expired
667
+
668
+ # If you've set a custom TTL in mcp.json, pass it explicitly β€” CLI commands
669
+ # run as a separate process and don't inherit env vars from mcp.json
670
+ VIDEO_MCP_CACHE_TTL_MINUTES=10 vmcp cache clear:expired
671
+
672
+ # Delete ALL cache entries β€” prompts for confirmation
673
+ vmcp cache clear:all
674
+ vmcp cache clear:all --yes # skip confirmation
675
+ ```
676
+
677
+ ### Video Summarization
678
+
679
+ | Variable | Description | Default |
680
+ | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------- |
681
+ | `VIDEO_MCP_MAX_FRAMES` | Max keyframes for `summarize_video` (GLM/Qwen/MiMo always; Kimi only when video > 100 MB; Gemini always uploads full video). **Free**: clamped 5–50. **Pro**: default 100; set `0` for uncapped. Either way, trailing frames are automatically dropped if the total payload exceeds the provider's size limit (12 MB for GLM, 10 MB for Qwen/MiMo, 80 MB for Kimi's frame fallback). | `50` free / `100` pro |
682
+
683
+ ### Qwen
684
+
685
+ | Variable | Description | Default |
686
+ | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- |
687
+ | `QWEN_BASE_URL` | Override the DashScope API endpoint. Use for regional routing: Singapore (default), Virginia (`https://dashscope-us.aliyuncs.com/compatible-mode/v1/chat/completions`), or Beijing (`https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions`). | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions` |
688
+ | `QWEN_REQUEST_TIMEOUT_MS` | Client-side timeout (ms) for Qwen API requests. DashScope applies a server-side limit of ~5 minutes, so values above ~285000 are unlikely to help. Increase to e.g. `270000` for extra headroom on large frame payloads. | `240000` (4 min) |
689
+
690
+ ### GLM
266
691
 
267
- | Variable | Description | Required |
268
- | ---------------------------- | --------------------------------- | ---------------------------------------- |
269
- | `MOONSHOT_API_KEY` | Moonshot AI API key for Kimi K2.5 | Optional (if using GLM) |
270
- | `Z_AI_API_KEY` | Z.AI API key for GLM-4.6V | Optional (if using Kimi) |
271
- | `VIDEO_MCP_DEFAULT_PROVIDER` | Default backend (`glm`, `kimi`) | Optional (default: `glm`) |
272
- | `VIDEO_MCP_MAX_FRAMES` | Max frames for summarization | Optional (default: 20; clamped to 5-100) |
692
+ | Variable | Description | Default |
693
+ | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- |
694
+ | `GLM_REQUEST_TIMEOUT_MS` | Client-side timeout (ms) for GLM API requests. Z.AI has no hard server-side streaming timeout, so raising this value (e.g. `480000` for 8 minutes) can help when analysing long videos with many frames. | `240000` (4 min) |
273
695
 
274
- ### Example Configuration
696
+ ### MiMo
697
+
698
+ | Variable | Description | Default |
699
+ | ------------------------- | ------------------------------------------------------------------------------ | ---------------- |
700
+ | `MIMO_REQUEST_TIMEOUT_MS` | Client-side timeout (ms) for MiMo API requests. Default: `240000` (4 minutes). | `240000` (4 min) |
701
+
702
+ ### yt-dlp (Platform Downloads)
703
+
704
+ | Variable | Description | Default |
705
+ | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
706
+ | `YT_DLP_MAX_RESOLUTION` | Max video height (px) for yt-dlp downloads. Free users capped at 720. | `720` |
707
+ | `YT_DLP_PATH` | Path to a custom yt-dlp binary. If unset, the server auto-detects common installs first and then uses the bundled binary from `youtube-dl-exec` if it is present. | β€” |
708
+ | `YT_DLP_COOKIES_FILE` | Path to a Netscape-format cookies file for authenticated downloads (age-restricted, private videos). | β€” |
709
+ | `YT_DLP_IMPERSONATE` | Browser to impersonate (e.g. `chrome`, `firefox`). Needed for sites like Udemy that block non-browser user-agents; requires a yt-dlp build with `curl_cffi` support. | β€” |
710
+
711
+ ### Pro
712
+
713
+ | Variable | Description | Default |
714
+ | ----------------------- | --------------------------------------------------------------------------------------------------------------------- | ------- |
715
+ | `VIDEO_MCP_LICENSE_KEY` | Pro license key. Unlocks extended frames, platform downloads beyond YouTube, and resolution above 720p. Valid 1 year. | β€” |
716
+
717
+ ### Complete Example Config
718
+
719
+ > **Warning:** This file contains sensitive API keys. **Never commit it to source control.** Make sure `.vscode/mcp.json` is in your `.gitignore`.
275
720
 
276
721
  ```json
277
722
  {
@@ -280,77 +725,215 @@ Ask Copilot Chat:
280
725
  "type": "stdio",
281
726
  "command": "video-context-mcp",
282
727
  "env": {
728
+ "GEMINI_API_KEY": "your-gemini-key",
283
729
  "Z_AI_API_KEY": "your-zai-key",
730
+ "DASHSCOPE_API_KEY": "your-dashscope-key",
284
731
  "MOONSHOT_API_KEY": "your-moonshot-key",
285
- "VIDEO_MCP_DEFAULT_PROVIDER": "glm",
286
- "VIDEO_MCP_MAX_FRAMES": "20"
732
+ "MIMO_API_KEY": "your-mimo-key",
733
+ "DEEPGRAM_API_KEY": "your-deepgram-key",
734
+ "ASSEMBLYAI_API_KEY": "your-assemblyai-key",
735
+ "GROQ_API_KEY": "your-groq-key",
736
+ "VIDEO_MCP_DEFAULT_PROVIDER": "gemini",
737
+ "AUDIO_MCP_DEFAULT_PROVIDER": "deepgram",
738
+ "AUDIO_ENHANCE_VIDEO_ANALYSIS": "auto",
739
+ "VIDEO_MCP_CACHE_TTL_MINUTES": "43200",
740
+ "VIDEO_MCP_CACHE_MAX_ENTRIES": "100",
741
+ "VIDEO_MCP_CACHE_MAX_MB": "5120",
742
+ "VIDEO_MCP_MAX_FRAMES": "50",
743
+ "YT_DLP_MAX_RESOLUTION": "720",
744
+ "YT_DLP_PATH": "/usr/local/bin/yt-dlp",
745
+ "YT_DLP_COOKIES_FILE": "/path/to/cookies.txt",
746
+ "YT_DLP_IMPERSONATE": "chrome",
747
+ "AWS_S3_BUCKET": "your-globally-unique-bucket-name",
748
+ "AWS_ACCESS_KEY_ID": "AKIA...",
749
+ "AWS_SECRET_ACCESS_KEY": "your-secret-key",
750
+ "AWS_REGION": "us-east-1",
751
+ "AWS_S3_RELAY_CLEANUP": "true",
752
+ "QWEN_BASE_URL": "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions",
753
+ "QWEN_REQUEST_TIMEOUT_MS": "240000",
754
+ "GLM_REQUEST_TIMEOUT_MS": "240000",
755
+ "MIMO_REQUEST_TIMEOUT_MS": "240000",
756
+ "VIDEO_MCP_LICENSE_KEY": "your-license-key"
287
757
  }
288
758
  }
289
759
  }
290
760
  }
291
761
  ```
292
762
 
293
- ## Development
763
+ > Remove or omit any variables you don't need β€” unset optional vars fall back to their defaults.
764
+
765
+ <details>
766
+ <summary><strong>Tip: install yt-dlp system-wide for faster startup</strong></summary>
767
+
768
+ The server first checks `YT_DLP_PATH`, then common system install locations, and then the bundled yt-dlp binary shipped by `youtube-dl-exec`.
769
+
770
+ If you use platform URLs often, install yt-dlp directly:
294
771
 
295
772
  ```bash
296
- # Install dependencies
297
- npm install
773
+ brew install yt-dlp # macOS
774
+ winget install yt-dlp # Windows
775
+ pip install yt-dlp # cross-platform
776
+ ```
298
777
 
299
- # Run in development (auto-restart on changes)
300
- npm run dev
778
+ Then set `YT_DLP_PATH` to the installed binary. If you installed `video-context-mcp-server` globally and the bundled binary is missing, repair it with:
301
779
 
302
- # Build for production
303
- npm run build
780
+ ```bash
781
+ npm rebuild youtube-dl-exec
782
+ ```
304
783
 
305
- # Run type checking
306
- npm run type-check
784
+ To skip the bundled download at install time:
307
785
 
308
- # Run linter
309
- npm run lint
786
+ ```bash
787
+ YOUTUBE_DL_SKIP_DOWNLOAD=true npm install -g video-context-mcp-server
788
+ ```
789
+
790
+ </details>
791
+
792
+ <details>
793
+ <summary><strong>Tip: use cookies for age-restricted or private videos</strong></summary>
794
+
795
+ Some videos require authentication. Export cookies from your browser and point yt-dlp at them.
796
+
797
+ **Step 1 β€” Export cookies.** Install [Get cookies.txt LOCALLY](https://chromewebstore.google.com/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) (Chrome/Edge) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (Firefox). Navigate to the platform while logged in, export in Netscape format, and save the file (e.g. `~/cookies-youtube.txt`).
798
+
799
+ **Step 2 β€” Configure.** Add `YT_DLP_COOKIES_FILE` to your MCP config `env` block:
800
+
801
+ ```jsonc
802
+ {
803
+ "env": {
804
+ "YT_DLP_COOKIES_FILE": "C:/Users/you/cookies-youtube.txt",
805
+ },
806
+ }
807
+ ```
808
+
809
+ Restart the MCP server. **Keep your cookies file private** β€” never commit it to source control.
810
+
811
+ </details>
812
+
813
+ ---
814
+
815
+ ## Pro
816
+
817
+ The free tier covers all core functionality. A **pro license** unlocks three power features:
818
+
819
+ | Feature | Free | Pro |
820
+ | ------------------------------------------------ | ------------------- | --------------------------------- |
821
+ | πŸ–ΌοΈ Extended frame extraction (`summarize_video`) | Capped at 50 frames | Default 100; set `0` for uncapped |
822
+ | 🌐 Platform video downloads | YouTube only | Almost all video platforms |
823
+ | πŸ“Ί Download resolution | Capped at 720p | Uncapped |
824
+
825
+ > Local files and direct `http(s)://` video URLs work for all users β€” the platform gate only applies to yt-dlp URLs (YouTube, Vimeo, TikTok, Bilibili, etc.).
826
+
827
+ When a free-tier limit is reached, the tool falls back gracefully with a notice β€” it **never hard-blocks**.
828
+
829
+ ### Getting a License
830
+
831
+ License keys are valid for **1 year**.
832
+
833
+ πŸš€ LAUNCH PROMO: Get PRO for Free!
834
+
835
+ For a limited time, you can get a PRO license for free. Just enter 0 in the amount field. If you'd like to support the development, feel free to enter any amount you chooseβ€”any contribution is greatly appreciated!
836
+
837
+ **[Purchase a Pro License on Gumroad](https://jeromekph.gumroad.com/l/video-context-mcp)**
838
+
839
+ Add `VIDEO_MCP_LICENSE_KEY` to your MCP config `env` block and restart the server:
840
+
841
+ ```json
842
+ {
843
+ "env": {
844
+ "VIDEO_MCP_LICENSE_KEY": "<your-license-key>"
845
+ }
846
+ }
847
+ ```
310
848
 
311
- # Run automated tests
312
- npm run test
849
+ ---
313
850
 
314
- # Run tests in watch mode
315
- npm run test:watch
851
+ ## Troubleshooting
316
852
 
317
- # Run tests with coverage
318
- npm run test:coverage
853
+ - **`video-context-mcp: command not found`** (or `vmcp: command not found`) β€” Make sure Node.js is installed (`node -v`). Re-run `npm install -g video-context-mcp-server`, or use `npx -y video-context-mcp-server@latest` if global binaries aren't on PATH.
319
854
 
320
- # Format all files
321
- npm run format
855
+ - **MCP server not appearing** β€” Restart the client app after config changes. Validate JSON syntax. For Claude Code, verify with `claude mcp list`.
322
856
 
323
- # Check formatting only
324
- npm run format:check
857
+ - **Missing API key errors** β€” Set only the keys for providers you use. When a key is missing, tools automatically fall back to the next available provider and include a notice in the response.
325
858
 
326
- # Lint + Type-check + Format + Build
327
- npm run ltfb
859
+ ---
860
+
861
+ ## Development
862
+
863
+ > This section is for contributors building from source. If you're just using the server, follow the [Quick Start](#quick-start) instead.
864
+
865
+ ### From Source
866
+
867
+ ```bash
868
+ git clone <repo-url> && cd video-mcp
869
+ npm install
870
+ npm run build
328
871
  ```
329
872
 
330
- ## Architecture
873
+ Use this `.vscode/mcp.json` to run the local build (never commit this file):
331
874
 
875
+ ```json
876
+ {
877
+ "servers": {
878
+ "videoMcp": {
879
+ "type": "stdio",
880
+ "command": "node",
881
+ "args": ["${workspaceFolder}/dist/index.js"],
882
+ "env": {
883
+ "GEMINI_API_KEY": "your-gemini-key",
884
+ "Z_AI_API_KEY": "your-zai-key",
885
+ "DASHSCOPE_API_KEY": "your-dashscope-key",
886
+ "MOONSHOT_API_KEY": "your-moonshot-key",
887
+ "MIMO_API_KEY": "your-mimo-key",
888
+ "VIDEO_MCP_DEFAULT_PROVIDER": "gemini",
889
+ "DEEPGRAM_API_KEY": "your-deepgram-key",
890
+ "ASSEMBLYAI_API_KEY": "your-assemblyai-key",
891
+ "GROQ_API_KEY": "your-groq-key",
892
+ "AUDIO_MCP_DEFAULT_PROVIDER": "deepgram",
893
+ "AUDIO_ENHANCE_VIDEO_ANALYSIS": "auto",
894
+ "VIDEO_MCP_CACHE_TTL_MINUTES": "43200",
895
+ "VIDEO_MCP_CACHE_MAX_ENTRIES": "100",
896
+ "VIDEO_MCP_CACHE_MAX_MB": "5120",
897
+ "VIDEO_MCP_MAX_FRAMES": "0",
898
+ "YT_DLP_MAX_RESOLUTION": "720",
899
+ "YT_DLP_PATH": "/usr/local/bin/yt-dlp",
900
+ "YT_DLP_COOKIES_FILE": "/path/to/cookies.txt",
901
+ "YT_DLP_IMPERSONATE": "chrome",
902
+ "VIDEO_MCP_LICENSE_KEY": "your-license-key"
903
+ }
904
+ }
905
+ }
906
+ }
332
907
  ```
333
- video-mcp/
334
- β”œβ”€β”€ src/
335
- β”‚ β”œβ”€β”€ index.ts # MCP server entry point
336
- β”‚ β”œβ”€β”€ tools/ # MCP tool implementations
337
- β”‚ β”œβ”€β”€ services/ # Backend clients (Kimi, GLM, ffmpeg)
338
- β”‚ └── utils/ # Helpers (temp files, base64)
339
- β”œβ”€β”€ .vscode/
340
- β”‚ └── mcp.json # VS Code MCP configuration
341
- β”œβ”€β”€ docs/
342
- β”‚ └── technical/ # Technical documentation
343
- └── .github/
344
- └── copilot-instructions.md # Copilot AI assistant guidelines
908
+
909
+ ### Debugging in VS Code
910
+
911
+ Add a `dev` block to `.vscode/mcp.json` for auto-restart and debug integration:
912
+
913
+ ```jsonc
914
+ "dev": {
915
+ "watch": "src/**/*.ts",
916
+ "debug": { "type": "node" }
917
+ }
345
918
  ```
346
919
 
920
+ This causes verbose MCP protocol logs in **Output β†’ MCP: videoMcp** β€” that's expected. Remove `dev.debug` or the full `dev` block to reduce noise.
921
+
922
+ ---
923
+
347
924
  ## License
348
925
 
349
- MIT
926
+ Proprietary β€” All Rights Reserved. No part of this software may be copied, modified, distributed, or used without explicit written permission from the author.
350
927
 
351
928
  ## Credits
352
929
 
353
930
  - [MCP SDK](https://github.com/modelcontextprotocol/typescript-sdk) by Anthropic
354
931
  - [Kimi K2.5](https://github.com/MoonshotAI/Kimi-K2.5) by Moonshot AI
355
932
  - [GLM-4.6V](https://docs.z.ai/guides/vlm/glm-4.6v) by Z.AI
933
+ - [Qwen3.6](https://bailian.console.alibabacloud.com/ap-southeast-1/) by Alibaba Cloud
934
+ - [MiMo-V2 Omni](https://platform.xiaomimimo.com/) by Xiaomi
935
+ - [Deepgram](https://www.deepgram.com/) for audio transcription
936
+ - [AssemblyAI](https://www.assemblyai.com/) for audio transcription
937
+ - [Groq/Whisper](https://console.groq.com/) for audio transcription
938
+ - [Gemini](https://aistudio.google.com/app/apikey) for audio transcription and as a fallback provider
356
939
  - [ffmpeg](https://ffmpeg.org/) for video processing