video-context-mcp-server 1.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +380 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +125 -0
- package/dist/index.js.map +1 -0
- package/dist/services/ffmpeg.d.ts +47 -0
- package/dist/services/ffmpeg.d.ts.map +1 -0
- package/dist/services/ffmpeg.js +170 -0
- package/dist/services/ffmpeg.js.map +1 -0
- package/dist/services/geminiClient.d.ts +47 -0
- package/dist/services/geminiClient.d.ts.map +1 -0
- package/dist/services/geminiClient.js +130 -0
- package/dist/services/geminiClient.js.map +1 -0
- package/dist/services/glmClient.d.ts +50 -0
- package/dist/services/glmClient.d.ts.map +1 -0
- package/dist/services/glmClient.js +196 -0
- package/dist/services/glmClient.js.map +1 -0
- package/dist/services/kimiClient.d.ts +45 -0
- package/dist/services/kimiClient.d.ts.map +1 -0
- package/dist/services/kimiClient.js +152 -0
- package/dist/services/kimiClient.js.map +1 -0
- package/dist/services/providerRouter.d.ts +30 -0
- package/dist/services/providerRouter.d.ts.map +1 -0
- package/dist/services/providerRouter.js +62 -0
- package/dist/services/providerRouter.js.map +1 -0
- package/dist/tools/analyzeVideo.d.ts +18 -0
- package/dist/tools/analyzeVideo.d.ts.map +1 -0
- package/dist/tools/analyzeVideo.js +118 -0
- package/dist/tools/analyzeVideo.js.map +1 -0
- package/dist/tools/extractFrames.d.ts +22 -0
- package/dist/tools/extractFrames.d.ts.map +1 -0
- package/dist/tools/extractFrames.js +82 -0
- package/dist/tools/extractFrames.js.map +1 -0
- package/dist/tools/getVideoInfo.d.ts +18 -0
- package/dist/tools/getVideoInfo.d.ts.map +1 -0
- package/dist/tools/getVideoInfo.js +52 -0
- package/dist/tools/getVideoInfo.js.map +1 -0
- package/dist/tools/searchTimestamp.d.ts +25 -0
- package/dist/tools/searchTimestamp.d.ts.map +1 -0
- package/dist/tools/searchTimestamp.js +146 -0
- package/dist/tools/searchTimestamp.js.map +1 -0
- package/dist/tools/summarizeVideo.d.ts +18 -0
- package/dist/tools/summarizeVideo.d.ts.map +1 -0
- package/dist/tools/summarizeVideo.js +159 -0
- package/dist/tools/summarizeVideo.js.map +1 -0
- package/dist/utils/base64.d.ts +35 -0
- package/dist/utils/base64.d.ts.map +1 -0
- package/dist/utils/base64.js +50 -0
- package/dist/utils/base64.js.map +1 -0
- package/dist/utils/logger.d.ts +23 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +34 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/tempFiles.d.ts +20 -0
- package/dist/utils/tempFiles.d.ts.map +1 -0
- package/dist/utils/tempFiles.js +31 -0
- package/dist/utils/tempFiles.js.map +1 -0
- package/dist/utils/videoUtils.d.ts +19 -0
- package/dist/utils/videoUtils.d.ts.map +1 -0
- package/dist/utils/videoUtils.js +38 -0
- package/dist/utils/videoUtils.js.map +1 -0
- package/package.json +62 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kimiClient.d.ts","sourceRoot":"","sources":["../../src/services/kimiClient.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAQ;gBAEV,OAAO,EAAE,iBAAiB;IAOtC;;;;OAIG;IACG,WAAW,CACf,IAAI,EAAE,OAAO,GACZ,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAa3D;;;;;OAKG;IACG,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBrE;;OAEG;IACG,wBAAwB,CAC5B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QACT,SAAS,EAAE,MAAM,CAAA;QACjB,MAAM,EAAE,MAAM,CAAA;KACf,CAAC;IAuCF;;OAEG;IACG,aAAa,CACjB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EACjD,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC;IA6BlB,OAAO,CAAC,6BAA6B;CA6BtC;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,UAAU,GAAG,IAAI,CAMpD"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
/**
|
|
3
|
+
* Kimi K2.5 Client
|
|
4
|
+
* Handles video upload and analysis using Moonshot AI's API
|
|
5
|
+
*/
|
|
6
|
+
const KIMI_BASE_URL = 'https://api.moonshot.ai/v1';
|
|
7
|
+
export class KimiClient {
|
|
8
|
+
client;
|
|
9
|
+
constructor(options) {
|
|
10
|
+
this.client = new OpenAI({
|
|
11
|
+
apiKey: options.apiKey,
|
|
12
|
+
baseURL: options.baseURL || KIMI_BASE_URL,
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Upload a video file to Moonshot's storage
|
|
17
|
+
* @param file - The video file (Buffer or ReadableStream)
|
|
18
|
+
* @returns File object with ID for use with ms:// protocol
|
|
19
|
+
*/
|
|
20
|
+
async uploadVideo(file) {
|
|
21
|
+
const fileObject = await this.client.files.create({
|
|
22
|
+
file: file,
|
|
23
|
+
purpose: 'video',
|
|
24
|
+
});
|
|
25
|
+
return {
|
|
26
|
+
id: fileObject.id,
|
|
27
|
+
object: fileObject.object,
|
|
28
|
+
purpose: fileObject.purpose,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Analyze a video using the uploaded file ID
|
|
33
|
+
* @param fileId - The file ID returned from uploadVideo()
|
|
34
|
+
* @param question - The question to ask about the video
|
|
35
|
+
* @returns The AI analysis response
|
|
36
|
+
*/
|
|
37
|
+
async analyzeVideo(fileId, question) {
|
|
38
|
+
const response = await this.client.chat.completions.create({
|
|
39
|
+
model: 'kimi-k2.5',
|
|
40
|
+
messages: [
|
|
41
|
+
{
|
|
42
|
+
role: 'user',
|
|
43
|
+
content: [
|
|
44
|
+
{
|
|
45
|
+
type: 'video_url',
|
|
46
|
+
video_url: { url: `ms://${fileId}` },
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
type: 'text',
|
|
50
|
+
text: question,
|
|
51
|
+
},
|
|
52
|
+
],
|
|
53
|
+
},
|
|
54
|
+
],
|
|
55
|
+
});
|
|
56
|
+
return this.extractTextFromMessageContent(response.choices?.[0]?.message?.content);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Analyze video with thinking mode enabled
|
|
60
|
+
*/
|
|
61
|
+
async analyzeVideoWithThinking(fileId, question) {
|
|
62
|
+
const response = await this.client.chat.completions.create({
|
|
63
|
+
model: 'kimi-k2.5',
|
|
64
|
+
messages: [
|
|
65
|
+
{
|
|
66
|
+
role: 'user',
|
|
67
|
+
content: [
|
|
68
|
+
{
|
|
69
|
+
type: 'video_url',
|
|
70
|
+
video_url: { url: `ms://${fileId}` },
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
type: 'text',
|
|
74
|
+
text: question,
|
|
75
|
+
},
|
|
76
|
+
],
|
|
77
|
+
},
|
|
78
|
+
],
|
|
79
|
+
});
|
|
80
|
+
const message = response.choices?.[0]?.message;
|
|
81
|
+
const reasoning = typeof message?.reasoning_content === 'string' &&
|
|
82
|
+
message.reasoning_content.length > 0
|
|
83
|
+
? message.reasoning_content
|
|
84
|
+
: 'No reasoning content returned by Kimi.';
|
|
85
|
+
return {
|
|
86
|
+
reasoning,
|
|
87
|
+
answer: this.extractTextFromMessageContent(message?.content),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Analyze multiple images (e.g., extracted frames)
|
|
92
|
+
*/
|
|
93
|
+
async analyzeImages(images, prompt) {
|
|
94
|
+
const imageContent = images.map((image) => ({
|
|
95
|
+
type: 'image_url',
|
|
96
|
+
image_url: {
|
|
97
|
+
url: `data:${image.mimeType};base64,${image.data}`,
|
|
98
|
+
},
|
|
99
|
+
}));
|
|
100
|
+
const response = await this.client.chat.completions.create({
|
|
101
|
+
model: 'kimi-k2.5',
|
|
102
|
+
messages: [
|
|
103
|
+
{
|
|
104
|
+
role: 'user',
|
|
105
|
+
content: [
|
|
106
|
+
...imageContent,
|
|
107
|
+
{
|
|
108
|
+
type: 'text',
|
|
109
|
+
text: prompt,
|
|
110
|
+
},
|
|
111
|
+
],
|
|
112
|
+
},
|
|
113
|
+
],
|
|
114
|
+
});
|
|
115
|
+
return this.extractTextFromMessageContent(response.choices?.[0]?.message?.content);
|
|
116
|
+
}
|
|
117
|
+
extractTextFromMessageContent(content) {
|
|
118
|
+
if (typeof content === 'string') {
|
|
119
|
+
return content;
|
|
120
|
+
}
|
|
121
|
+
if (Array.isArray(content)) {
|
|
122
|
+
const textParts = content
|
|
123
|
+
.map((part) => {
|
|
124
|
+
if (typeof part === 'object' &&
|
|
125
|
+
part !== null &&
|
|
126
|
+
'type' in part &&
|
|
127
|
+
part.type === 'text' &&
|
|
128
|
+
'text' in part) {
|
|
129
|
+
const text = part.text;
|
|
130
|
+
return typeof text === 'string' ? text : '';
|
|
131
|
+
}
|
|
132
|
+
return '';
|
|
133
|
+
})
|
|
134
|
+
.filter((text) => text.length > 0);
|
|
135
|
+
if (textParts.length > 0) {
|
|
136
|
+
return textParts.join('\n');
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return 'No response content returned by Kimi.';
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Create a Kimi client instance from environment variables
|
|
144
|
+
*/
|
|
145
|
+
export function createKimiClient() {
|
|
146
|
+
const apiKey = process.env.MOONSHOT_API_KEY;
|
|
147
|
+
if (!apiKey) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
return new KimiClient({ apiKey });
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=kimiClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kimiClient.js","sourceRoot":"","sources":["../../src/services/kimiClient.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAA;AAE3B;;;GAGG;AAEH,MAAM,aAAa,GAAG,4BAA4B,CAAA;AAOlD,MAAM,OAAO,UAAU;IACb,MAAM,CAAQ;IAEtB,YAAY,OAA0B;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,aAAa;SAC1C,CAAC,CAAA;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CACf,IAAa;QAEb,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAChD,IAAI,EAAE,IAAW;YACjB,OAAO,EAAE,OAAc;SACjB,CAAC,CAAA;QAET,OAAO;YACL,EAAE,EAAE,UAAU,CAAC,EAAE;YACjB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE,UAAU,CAAC,OAAO;SAC5B,CAAA;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,QAAgB;QACjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACzD,KAAK,EAAE,WAAW;YAClB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,WAAW;4BACjB,SAAS,EAAE,EAAE,GAAG,EAAE,QAAQ,MAAM,EAAE,EAAE;yBACrC;wBACD;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF;aACF;SACK,CAAC,CAAA;QAET,OAAO,IAAI,CAAC,6BAA6B,CACvC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CACxC,CAAA;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,wBAAwB,CAC5B,MAAc,EACd,QAAgB;QAKhB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACzD,KAAK,EAAE,WAAW;YAClB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,WAAW;4BACjB,SAAS,EAAE,EAAE,GAAG,EAAE,QAAQ,MAAM,EAAE,EAAE;yBACrC;wBACD;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF;aACF;SACK,CAAC,CAAA;QAET,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAK1B,CAAA;QAEb,MAAM,SAAS,GACb,OAAO,OAAO,EAAE,iBAAiB,KAAK,QAAQ;YAC9C,OAAO,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC;YAClC,CAAC,CAAC,OAAO,CAAC,iBAAiB;YAC3B,CAAC,CAAC,wCAAwC,CAAA;QAE9C,OAAO;YACL,SAAS;YACT,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,OAAO,EAAE,OAAO,CAAC;SAC7D,CAAA;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,MAAiD,EACjD,MAAc;QAEd,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,EAAE,WAAW;YACjB,SAAS,EAAE;gBACT,GAAG,EAAE,QAAQ,KAAK,CAAC,QAAQ,WAAW,KAAK,CAAC,IAAI,EAAE;aACnD;SACF,CAAC,CAAC,CAAA;QAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YACzD,KAAK,EAAE,WAAW;YAClB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE;wBACP,GAAG,YAAY;wBACf;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,MAAM;yBACb;qBACF;iBACF;aACF;SACK,CAAC,CAAA;QAET,OAAO,IAAI,CAAC,6BAA6B,CACvC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CACxC,CAAA;IACH,CAAC;IAEO,6BAA6B,CAAC,OAAgB;QACpD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,OAAO,CAAA;QAChB,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,OAAO;iBACtB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IACE,OAAO,IAAI,KAAK,QAAQ;oBACxB,IAAI,KAAK,IAAI;oBACb,MAAM,IAAI,IAAI;oBACb,IAA0B,CAAC,IAAI,KAAK,MAAM;oBAC3C,MAAM,IAAI,IAAI,EACd,CAAC;oBACD,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAA;oBAC9C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAA;gBAC7C,CAAC;gBACD,OAAO,EAAE,CAAA;YACX,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YAEpC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC7B,CAAC;QACH,CAAC;QAED,OAAO,uCAAuC,CAAA;IAChD,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAA;IAC3C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,IAAI,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;AACnC,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider Selection
|
|
3
|
+
* Determines which AI backend to use based on user preference
|
|
4
|
+
*/
|
|
5
|
+
export type VideoProvider = 'glm' | 'kimi' | 'gemini';
|
|
6
|
+
/**
|
|
7
|
+
* Validate that the selected provider has a configured API key
|
|
8
|
+
* @param provider - User-specified provider ('glm', 'kimi', or 'gemini')
|
|
9
|
+
* @param hasKimiKey - Whether Kimi API key is available
|
|
10
|
+
* @param hasGLMKey - Whether GLM API key is available
|
|
11
|
+
* @param hasGeminiKey - Whether Gemini API key is available
|
|
12
|
+
* @returns The validated provider
|
|
13
|
+
*/
|
|
14
|
+
export declare function selectProvider(provider: VideoProvider, hasKimiKey: boolean, hasGLMKey: boolean, hasGeminiKey?: boolean): VideoProvider;
|
|
15
|
+
/**
|
|
16
|
+
* Resolve the provider, with optional fallback behavior.
|
|
17
|
+
* - If requested provider is glm but GLM key is missing, fallback to kimi, then gemini.
|
|
18
|
+
* - If requested provider is kimi but Kimi key is missing and Gemini key exists, fallback to gemini.
|
|
19
|
+
* - If requested provider is gemini, no fallback is applied (it's the last resort).
|
|
20
|
+
*/
|
|
21
|
+
export declare function resolveProviderWithFallback(provider: VideoProvider, hasKimiKey: boolean, hasGLMKey: boolean, hasGeminiKey?: boolean): VideoProvider;
|
|
22
|
+
/**
|
|
23
|
+
* Get the default provider from environment variable
|
|
24
|
+
*/
|
|
25
|
+
export declare function getDefaultProvider(): VideoProvider;
|
|
26
|
+
/**
|
|
27
|
+
* Get the maximum number of frames to extract for summarization
|
|
28
|
+
*/
|
|
29
|
+
export declare function getMaxFrames(): number;
|
|
30
|
+
//# sourceMappingURL=providerRouter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"providerRouter.d.ts","sourceRoot":"","sources":["../../src/services/providerRouter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAA;AAErD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,aAAa,EACvB,UAAU,EAAE,OAAO,EACnB,SAAS,EAAE,OAAO,EAClB,YAAY,GAAE,OAAe,GAC5B,aAAa,CAiBf;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,aAAa,EACvB,UAAU,EAAE,OAAO,EACnB,SAAS,EAAE,OAAO,EAClB,YAAY,GAAE,OAAe,GAC5B,aAAa,CAUf;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,aAAa,CAUlD;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAGrC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider Selection
|
|
3
|
+
* Determines which AI backend to use based on user preference
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Validate that the selected provider has a configured API key
|
|
7
|
+
* @param provider - User-specified provider ('glm', 'kimi', or 'gemini')
|
|
8
|
+
* @param hasKimiKey - Whether Kimi API key is available
|
|
9
|
+
* @param hasGLMKey - Whether GLM API key is available
|
|
10
|
+
* @param hasGeminiKey - Whether Gemini API key is available
|
|
11
|
+
* @returns The validated provider
|
|
12
|
+
*/
|
|
13
|
+
export function selectProvider(provider, hasKimiKey, hasGLMKey, hasGeminiKey = false) {
|
|
14
|
+
if (provider === 'gemini' && !hasGeminiKey) {
|
|
15
|
+
throw new Error('Gemini provider selected but GEMINI_API_KEY is not configured');
|
|
16
|
+
}
|
|
17
|
+
if (provider === 'kimi' && !hasKimiKey) {
|
|
18
|
+
throw new Error('Kimi K2.5 provider selected but MOONSHOT_API_KEY is not configured');
|
|
19
|
+
}
|
|
20
|
+
if (provider === 'glm' && !hasGLMKey) {
|
|
21
|
+
throw new Error('GLM-4.6V provider selected but Z_AI_API_KEY is not configured');
|
|
22
|
+
}
|
|
23
|
+
return provider;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the provider, with optional fallback behavior.
|
|
27
|
+
* - If requested provider is glm but GLM key is missing, fallback to kimi, then gemini.
|
|
28
|
+
* - If requested provider is kimi but Kimi key is missing and Gemini key exists, fallback to gemini.
|
|
29
|
+
* - If requested provider is gemini, no fallback is applied (it's the last resort).
|
|
30
|
+
*/
|
|
31
|
+
export function resolveProviderWithFallback(provider, hasKimiKey, hasGLMKey, hasGeminiKey = false) {
|
|
32
|
+
if (provider === 'glm' && !hasGLMKey) {
|
|
33
|
+
if (hasKimiKey)
|
|
34
|
+
return 'kimi';
|
|
35
|
+
if (hasGeminiKey)
|
|
36
|
+
return 'gemini';
|
|
37
|
+
}
|
|
38
|
+
if (provider === 'kimi' && !hasKimiKey && hasGeminiKey) {
|
|
39
|
+
return 'gemini';
|
|
40
|
+
}
|
|
41
|
+
return selectProvider(provider, hasKimiKey, hasGLMKey, hasGeminiKey);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Get the default provider from environment variable
|
|
45
|
+
*/
|
|
46
|
+
export function getDefaultProvider() {
|
|
47
|
+
const envProvider = process.env.VIDEO_MCP_DEFAULT_PROVIDER;
|
|
48
|
+
if (envProvider === 'glm' ||
|
|
49
|
+
envProvider === 'kimi' ||
|
|
50
|
+
envProvider === 'gemini') {
|
|
51
|
+
return envProvider;
|
|
52
|
+
}
|
|
53
|
+
return 'glm';
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Get the maximum number of frames to extract for summarization
|
|
57
|
+
*/
|
|
58
|
+
export function getMaxFrames() {
|
|
59
|
+
const envMaxFrames = parseInt(process.env.VIDEO_MCP_MAX_FRAMES || '20', 10);
|
|
60
|
+
return Math.max(5, Math.min(100, envMaxFrames)); // Clamp between 5 and 100
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=providerRouter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"providerRouter.js","sourceRoot":"","sources":["../../src/services/providerRouter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAuB,EACvB,UAAmB,EACnB,SAAkB,EAClB,eAAwB,KAAK;IAE7B,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CACb,+DAA+D,CAChE,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,KAAK,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,oEAAoE,CACrE,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,+DAA+D,CAChE,CAAA;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CACzC,QAAuB,EACvB,UAAmB,EACnB,SAAkB,EAClB,eAAwB,KAAK;IAE7B,IAAI,QAAQ,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,UAAU;YAAE,OAAO,MAAM,CAAA;QAC7B,IAAI,YAAY;YAAE,OAAO,QAAQ,CAAA;IACnC,CAAC;IACD,IAAI,QAAQ,KAAK,MAAM,IAAI,CAAC,UAAU,IAAI,YAAY,EAAE,CAAC;QACvD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED,OAAO,cAAc,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,CAAC,CAAA;AACtE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAA;IAC1D,IACE,WAAW,KAAK,KAAK;QACrB,WAAW,KAAK,MAAM;QACtB,WAAW,KAAK,QAAQ,EACxB,CAAC;QACD,OAAO,WAAW,CAAA;IACpB,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,IAAI,EAAE,EAAE,CAAC,CAAA;IAC3E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAA,CAAC,0BAA0B;AAC5E,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: analyze_video
|
|
3
|
+
* Ask questions about video content and get AI-powered answers
|
|
4
|
+
*/
|
|
5
|
+
export declare const analyzeVideoTool: (params: any) => Promise<{
|
|
6
|
+
content: {
|
|
7
|
+
type: "text";
|
|
8
|
+
text: string;
|
|
9
|
+
}[];
|
|
10
|
+
isError?: undefined;
|
|
11
|
+
} | {
|
|
12
|
+
isError: boolean;
|
|
13
|
+
content: {
|
|
14
|
+
type: "text";
|
|
15
|
+
text: string;
|
|
16
|
+
}[];
|
|
17
|
+
}>;
|
|
18
|
+
//# sourceMappingURL=analyzeVideo.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyzeVideo.d.ts","sourceRoot":"","sources":["../../src/tools/analyzeVideo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAqBH,eAAO,MAAM,gBAAgB,GAAU,QAAQ,GAAG;;;;;;;;;;;;EA4HjD,CAAA"}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: analyze_video
|
|
3
|
+
* Ask questions about video content and get AI-powered answers
|
|
4
|
+
*/
|
|
5
|
+
import * as fs from 'fs/promises';
|
|
6
|
+
import { createReadStream } from 'fs';
|
|
7
|
+
import { createGLMClient } from '../services/glmClient.js';
|
|
8
|
+
import { createKimiClient } from '../services/kimiClient.js';
|
|
9
|
+
import { createGeminiClient } from '../services/geminiClient.js';
|
|
10
|
+
import { getDefaultProvider, resolveProviderWithFallback, } from '../services/providerRouter.js';
|
|
11
|
+
import { bufferToBase64String } from '../utils/base64.js';
|
|
12
|
+
import { createTempDir, cleanupTempDir } from '../utils/tempFiles.js';
|
|
13
|
+
import { isRemoteUrl, normalizeVideoPath, downloadVideoToTemp, } from '../utils/videoUtils.js';
|
|
14
|
+
import { logProgress } from '../utils/logger.js';
|
|
15
|
+
export const analyzeVideoTool = async (params) => {
|
|
16
|
+
let tempDir = null;
|
|
17
|
+
try {
|
|
18
|
+
const { videoPath, question } = params;
|
|
19
|
+
const provider = params.provider ?? getDefaultProvider();
|
|
20
|
+
const glmClient = createGLMClient();
|
|
21
|
+
const kimiClient = createKimiClient();
|
|
22
|
+
const geminiClient = createGeminiClient();
|
|
23
|
+
let selectedProvider;
|
|
24
|
+
let answer;
|
|
25
|
+
if (isRemoteUrl(videoPath)) {
|
|
26
|
+
if (provider === 'kimi') {
|
|
27
|
+
throw new Error('Kimi provider does not support direct remote URLs. Use provider=glm, provider=gemini, or local file input.');
|
|
28
|
+
}
|
|
29
|
+
selectedProvider = resolveProviderWithFallback(provider, Boolean(kimiClient), Boolean(glmClient), Boolean(geminiClient));
|
|
30
|
+
if (selectedProvider === 'gemini') {
|
|
31
|
+
if (!geminiClient) {
|
|
32
|
+
throw new Error('Gemini client is unavailable. Set GEMINI_API_KEY.');
|
|
33
|
+
}
|
|
34
|
+
tempDir = await createTempDir();
|
|
35
|
+
await logProgress('Downloading remote video...');
|
|
36
|
+
const localPath = await downloadVideoToTemp(videoPath, tempDir);
|
|
37
|
+
await logProgress('Uploading to Gemini...');
|
|
38
|
+
const uploadedVideo = await geminiClient.uploadVideo(localPath);
|
|
39
|
+
await logProgress('Waiting for Gemini to process video...');
|
|
40
|
+
await geminiClient.waitForFileProcessing(uploadedVideo.name);
|
|
41
|
+
await logProgress('Analyzing video...');
|
|
42
|
+
answer = await geminiClient.analyzeVideo(uploadedVideo, question);
|
|
43
|
+
}
|
|
44
|
+
else if (selectedProvider === 'glm') {
|
|
45
|
+
if (!glmClient) {
|
|
46
|
+
throw new Error('GLM provider is required for URL video analysis, but Z_AI_API_KEY is not configured.');
|
|
47
|
+
}
|
|
48
|
+
await logProgress('Analyzing remote video...');
|
|
49
|
+
answer = await glmClient.analyzeVideoUrl(videoPath, question);
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
throw new Error('No suitable provider found for remote URL analysis.');
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
const resolvedPath = normalizeVideoPath(videoPath);
|
|
57
|
+
selectedProvider = resolveProviderWithFallback(provider, Boolean(kimiClient), Boolean(glmClient), Boolean(geminiClient));
|
|
58
|
+
if (selectedProvider === 'gemini') {
|
|
59
|
+
if (!geminiClient) {
|
|
60
|
+
throw new Error('Gemini client is unavailable. Set GEMINI_API_KEY.');
|
|
61
|
+
}
|
|
62
|
+
await logProgress('Uploading to Gemini...');
|
|
63
|
+
const uploadedVideo = await geminiClient.uploadVideo(resolvedPath);
|
|
64
|
+
await logProgress('Waiting for Gemini to process video...');
|
|
65
|
+
await geminiClient.waitForFileProcessing(uploadedVideo.name);
|
|
66
|
+
await logProgress('Analyzing video...');
|
|
67
|
+
answer = await geminiClient.analyzeVideo(uploadedVideo, question);
|
|
68
|
+
}
|
|
69
|
+
else if (selectedProvider === 'glm') {
|
|
70
|
+
if (!glmClient) {
|
|
71
|
+
throw new Error('GLM client is unavailable. Set Z_AI_API_KEY.');
|
|
72
|
+
}
|
|
73
|
+
await logProgress('Analyzing video...');
|
|
74
|
+
const videoBuffer = await fs.readFile(resolvedPath);
|
|
75
|
+
answer = await glmClient.analyzeVideoBase64(bufferToBase64String(videoBuffer), question);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
if (!kimiClient) {
|
|
79
|
+
throw new Error('Kimi client is unavailable. Set MOONSHOT_API_KEY.');
|
|
80
|
+
}
|
|
81
|
+
await logProgress('Uploading to Kimi...');
|
|
82
|
+
const uploadedVideo = await kimiClient.uploadVideo(createReadStream(resolvedPath));
|
|
83
|
+
await logProgress('Analyzing video...');
|
|
84
|
+
answer = await kimiClient.analyzeVideo(uploadedVideo.id, question);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (tempDir) {
|
|
88
|
+
await cleanupTempDir(tempDir);
|
|
89
|
+
}
|
|
90
|
+
return {
|
|
91
|
+
content: [
|
|
92
|
+
{
|
|
93
|
+
type: 'text',
|
|
94
|
+
text: answer,
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
type: 'text',
|
|
98
|
+
text: `Provider used: ${selectedProvider}`,
|
|
99
|
+
},
|
|
100
|
+
],
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
if (tempDir) {
|
|
105
|
+
await cleanupTempDir(tempDir).catch(() => { });
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
isError: true,
|
|
109
|
+
content: [
|
|
110
|
+
{
|
|
111
|
+
type: 'text',
|
|
112
|
+
text: `Error analyzing video: ${error instanceof Error ? error.message : String(error)}`,
|
|
113
|
+
},
|
|
114
|
+
],
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
//# sourceMappingURL=analyzeVideo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyzeVideo.js","sourceRoot":"","sources":["../../src/tools/analyzeVideo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChE,OAAO,EACL,kBAAkB,EAClB,2BAA2B,GAE5B,MAAM,+BAA+B,CAAA;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACrE,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAW,EAAE,EAAE;IACpD,IAAI,OAAO,GAAkB,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAA;QACtC,MAAM,QAAQ,GAAkB,MAAM,CAAC,QAAQ,IAAI,kBAAkB,EAAE,CAAA;QAEvE,MAAM,SAAS,GAAG,eAAe,EAAE,CAAA;QACnC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAA;QACrC,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAA;QAEzC,IAAI,gBAA+B,CAAA;QACnC,IAAI,MAAc,CAAA;QAElB,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACb,4GAA4G,CAC7G,CAAA;YACH,CAAC;YAED,gBAAgB,GAAG,2BAA2B,CAC5C,QAAQ,EACR,OAAO,CAAC,UAAU,CAAC,EACnB,OAAO,CAAC,SAAS,CAAC,EAClB,OAAO,CAAC,YAAY,CAAC,CACtB,CAAA;YAED,IAAI,gBAAgB,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;gBACtE,CAAC;gBACD,OAAO,GAAG,MAAM,aAAa,EAAE,CAAA;gBAC/B,MAAM,WAAW,CAAC,6BAA6B,CAAC,CAAA;gBAChD,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;gBAC/D,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;gBAC3C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;gBAC/D,MAAM,WAAW,CAAC,wCAAwC,CAAC,CAAA;gBAC3D,MAAM,YAAY,CAAC,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;gBAC5D,MAAM,WAAW,CAAC,oBAAoB,CAAC,CAAA;gBACvC,MAAM,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;YACnE,CAAC;iBAAM,IAAI,gBAAgB,KAAK,KAAK,EAAE,CAAC;gBACtC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;gBACH,CAAC;gBACD,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAA;gBAC9C,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;YAC/D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;YACxE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;YAElD,gBAAgB,GAAG,2BAA2B,CAC5C,QAAQ,EACR,OAAO,CAAC,UAAU,CAAC,EACnB,OAAO,CAAC,SAAS,CAAC,EAClB,OAAO,CAAC,YAAY,CAAC,CACtB,CAAA;YAED,IAAI,gBAAgB,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;gBACtE,CAAC;gBACD,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;gBAC3C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;gBAClE,MAAM,WAAW,CAAC,wCAAwC,CAAC,CAAA;gBAC3D,MAAM,YAAY,CAAC,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;gBAC5D,MAAM,WAAW,CAAC,oBAAoB,CAAC,CAAA;gBACvC,MAAM,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;YACnE,CAAC;iBAAM,IAAI,gBAAgB,KAAK,KAAK,EAAE,CAAC;gBACtC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;gBACjE,CAAC;gBACD,MAAM,WAAW,CAAC,oBAAoB,CAAC,CAAA;gBACvC,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;gBACnD,MAAM,GAAG,MAAM,SAAS,CAAC,kBAAkB,CACzC,oBAAoB,CAAC,WAAW,CAAC,EACjC,QAAQ,CACT,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;gBACtE,CAAC;gBACD,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;gBACzC,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,WAAW,CAChD,gBAAgB,CAAC,YAAY,CAAC,CAC/B,CAAA;gBACD,MAAM,WAAW,CAAC,oBAAoB,CAAC,CAAA;gBACvC,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;YACpE,CAAC;QACH,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;QAC/B,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,MAAM;iBACb;gBACD;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,kBAAkB,gBAAgB,EAAE;iBAC3C;aACF;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC/C,CAAC;QACD,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBACzF;aACF;SACF,CAAA;IACH,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: extract_frames
|
|
3
|
+
* Extract frames from a video at specific timestamps or intervals
|
|
4
|
+
*/
|
|
5
|
+
export declare const extractFramesTool: (params: any) => Promise<{
|
|
6
|
+
content: ({
|
|
7
|
+
type: "image";
|
|
8
|
+
data: string;
|
|
9
|
+
mimeType: string;
|
|
10
|
+
} | {
|
|
11
|
+
type: "text";
|
|
12
|
+
text: string;
|
|
13
|
+
})[];
|
|
14
|
+
isError?: undefined;
|
|
15
|
+
} | {
|
|
16
|
+
isError: boolean;
|
|
17
|
+
content: {
|
|
18
|
+
type: "text";
|
|
19
|
+
text: string;
|
|
20
|
+
}[];
|
|
21
|
+
}>;
|
|
22
|
+
//# sourceMappingURL=extractFrames.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractFrames.d.ts","sourceRoot":"","sources":["../../src/tools/extractFrames.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAiBH,eAAO,MAAM,iBAAiB,GAAU,QAAQ,GAAG;;;;;;;;;;;;;;;;EA8ElD,CAAA"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: extract_frames
|
|
3
|
+
* Extract frames from a video at specific timestamps or intervals
|
|
4
|
+
*/
|
|
5
|
+
import { extractFrameAt, extractFramesAtInterval, extractFramesAtTimestamps, extractFramesEvenly, } from '../services/ffmpeg.js';
|
|
6
|
+
import { bufferToBase64String } from '../utils/base64.js';
|
|
7
|
+
import { createTempDir, cleanupTempDir } from '../utils/tempFiles.js';
|
|
8
|
+
import { isRemoteUrl, normalizeVideoPath, downloadVideoToTemp, } from '../utils/videoUtils.js';
|
|
9
|
+
import { logProgress } from '../utils/logger.js';
|
|
10
|
+
export const extractFramesTool = async (params) => {
|
|
11
|
+
let tempDir = null;
|
|
12
|
+
try {
|
|
13
|
+
const { videoPath, mode, count, intervalSec, timestamps } = params;
|
|
14
|
+
let resolvedPath;
|
|
15
|
+
if (isRemoteUrl(videoPath)) {
|
|
16
|
+
tempDir = await createTempDir();
|
|
17
|
+
await logProgress('Downloading remote video...');
|
|
18
|
+
resolvedPath = await downloadVideoToTemp(videoPath, tempDir);
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
resolvedPath = normalizeVideoPath(videoPath);
|
|
22
|
+
}
|
|
23
|
+
await logProgress('Extracting frames...');
|
|
24
|
+
let frames = [];
|
|
25
|
+
if (mode === 'even') {
|
|
26
|
+
if (!count) {
|
|
27
|
+
throw new Error("'count' is required when mode is 'even'");
|
|
28
|
+
}
|
|
29
|
+
frames = await extractFramesEvenly(resolvedPath, count);
|
|
30
|
+
}
|
|
31
|
+
else if (mode === 'interval') {
|
|
32
|
+
if (!intervalSec) {
|
|
33
|
+
throw new Error("'intervalSec' is required when mode is 'interval'");
|
|
34
|
+
}
|
|
35
|
+
frames = await extractFramesAtInterval(resolvedPath, intervalSec);
|
|
36
|
+
}
|
|
37
|
+
else if (mode === 'timestamps') {
|
|
38
|
+
if (!Array.isArray(timestamps) || timestamps.length === 0) {
|
|
39
|
+
throw new Error("'timestamps' (non-empty array) is required when mode is 'timestamps'");
|
|
40
|
+
}
|
|
41
|
+
frames = await extractFramesAtTimestamps(resolvedPath, timestamps);
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
throw new Error(`Unsupported extraction mode: ${String(mode)}`);
|
|
45
|
+
}
|
|
46
|
+
if (frames.length === 0 &&
|
|
47
|
+
mode === 'timestamps' &&
|
|
48
|
+
Array.isArray(timestamps)) {
|
|
49
|
+
const fallbackFrames = await Promise.all(timestamps.map((timestamp) => extractFrameAt(resolvedPath, timestamp)));
|
|
50
|
+
frames = fallbackFrames;
|
|
51
|
+
}
|
|
52
|
+
if (tempDir)
|
|
53
|
+
await cleanupTempDir(tempDir);
|
|
54
|
+
return {
|
|
55
|
+
content: [
|
|
56
|
+
{
|
|
57
|
+
type: 'text',
|
|
58
|
+
text: `Extracted ${frames.length} frame(s) from ${videoPath} using ${mode} mode.`,
|
|
59
|
+
},
|
|
60
|
+
...frames.map((frameBuffer) => ({
|
|
61
|
+
type: 'image',
|
|
62
|
+
data: bufferToBase64String(frameBuffer),
|
|
63
|
+
mimeType: 'image/png',
|
|
64
|
+
})),
|
|
65
|
+
],
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (tempDir)
|
|
70
|
+
await cleanupTempDir(tempDir).catch(() => { });
|
|
71
|
+
return {
|
|
72
|
+
isError: true,
|
|
73
|
+
content: [
|
|
74
|
+
{
|
|
75
|
+
type: 'text',
|
|
76
|
+
text: `Error extracting frames: ${error instanceof Error ? error.message : String(error)}`,
|
|
77
|
+
},
|
|
78
|
+
],
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
//# sourceMappingURL=extractFrames.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractFrames.js","sourceRoot":"","sources":["../../src/tools/extractFrames.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACrE,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAEhD,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,EAAE,MAAW,EAAE,EAAE;IACrD,IAAI,OAAO,GAAkB,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,CAAA;QAElE,IAAI,YAAoB,CAAA;QACxB,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,OAAO,GAAG,MAAM,aAAa,EAAE,CAAA;YAC/B,MAAM,WAAW,CAAC,6BAA6B,CAAC,CAAA;YAChD,YAAY,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC9D,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC9C,CAAC;QAED,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;QACzC,IAAI,MAAM,GAAa,EAAE,CAAA;QAEzB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;YAC5D,CAAC;YACD,MAAM,GAAG,MAAM,mBAAmB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;QACzD,CAAC;aAAM,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;YACtE,CAAC;YACD,MAAM,GAAG,MAAM,uBAAuB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;QACnE,CAAC;aAAM,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1D,MAAM,IAAI,KAAK,CACb,sEAAsE,CACvE,CAAA;YACH,CAAC;YACD,MAAM,GAAG,MAAM,yBAAyB,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QACpE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjE,CAAC;QAED,IACE,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,IAAI,KAAK,YAAY;YACrB,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EACzB,CAAC;YACD,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAiB,EAAE,EAAE,CACnC,cAAc,CAAC,YAAY,EAAE,SAAS,CAAC,CACxC,CACF,CAAA;YACD,MAAM,GAAG,cAAc,CAAA;QACzB,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;QAE1C,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,aAAa,MAAM,CAAC,MAAM,kBAAkB,SAAS,UAAU,IAAI,QAAQ;iBAClF;gBACD,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;oBAC9B,IAAI,EAAE,OAAgB;oBACtB,IAAI,EAAE,oBAAoB,CAAC,WAAW,CAAC;oBACvC,QAAQ,EAAE,WAAW;iBACtB,CAAC,CAAC;aACJ;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC1D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBAC3F;aACF;SACF,CAAA;IACH,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: get_video_info
|
|
3
|
+
* Get video metadata
|
|
4
|
+
*/
|
|
5
|
+
export declare const getVideoInfoTool: (params: any) => Promise<{
|
|
6
|
+
content: {
|
|
7
|
+
type: "text";
|
|
8
|
+
text: string;
|
|
9
|
+
}[];
|
|
10
|
+
isError?: undefined;
|
|
11
|
+
} | {
|
|
12
|
+
isError: boolean;
|
|
13
|
+
content: {
|
|
14
|
+
type: "text";
|
|
15
|
+
text: string;
|
|
16
|
+
}[];
|
|
17
|
+
}>;
|
|
18
|
+
//# sourceMappingURL=getVideoInfo.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getVideoInfo.d.ts","sourceRoot":"","sources":["../../src/tools/getVideoInfo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH,eAAO,MAAM,gBAAgB,GAAU,QAAQ,GAAG;;;;;;;;;;;;EA8CjD,CAAA"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: get_video_info
|
|
3
|
+
* Get video metadata
|
|
4
|
+
*/
|
|
5
|
+
import { getVideoMetadata } from '../services/ffmpeg.js';
|
|
6
|
+
import { createTempDir, cleanupTempDir } from '../utils/tempFiles.js';
|
|
7
|
+
import { isRemoteUrl, normalizeVideoPath, downloadVideoToTemp, } from '../utils/videoUtils.js';
|
|
8
|
+
import { logProgress } from '../utils/logger.js';
|
|
9
|
+
export const getVideoInfoTool = async (params) => {
|
|
10
|
+
let tempDir = null;
|
|
11
|
+
try {
|
|
12
|
+
const { videoPath } = params;
|
|
13
|
+
let resolvedPath;
|
|
14
|
+
if (isRemoteUrl(videoPath)) {
|
|
15
|
+
tempDir = await createTempDir();
|
|
16
|
+
await logProgress('Downloading remote video...');
|
|
17
|
+
resolvedPath = await downloadVideoToTemp(videoPath, tempDir);
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
resolvedPath = normalizeVideoPath(videoPath);
|
|
21
|
+
}
|
|
22
|
+
await logProgress('Reading video metadata...');
|
|
23
|
+
const metadata = await getVideoMetadata(resolvedPath);
|
|
24
|
+
if (tempDir)
|
|
25
|
+
await cleanupTempDir(tempDir);
|
|
26
|
+
return {
|
|
27
|
+
content: [
|
|
28
|
+
{
|
|
29
|
+
type: 'text',
|
|
30
|
+
text: JSON.stringify({
|
|
31
|
+
videoPath,
|
|
32
|
+
...metadata,
|
|
33
|
+
}, null, 2),
|
|
34
|
+
},
|
|
35
|
+
],
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
if (tempDir)
|
|
40
|
+
await cleanupTempDir(tempDir).catch(() => { });
|
|
41
|
+
return {
|
|
42
|
+
isError: true,
|
|
43
|
+
content: [
|
|
44
|
+
{
|
|
45
|
+
type: 'text',
|
|
46
|
+
text: `Error getting video info: ${error instanceof Error ? error.message : String(error)}`,
|
|
47
|
+
},
|
|
48
|
+
],
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
//# sourceMappingURL=getVideoInfo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getVideoInfo.js","sourceRoot":"","sources":["../../src/tools/getVideoInfo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACrE,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAW,EAAE,EAAE;IACpD,IAAI,OAAO,GAAkB,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAA;QAE5B,IAAI,YAAoB,CAAA;QACxB,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,OAAO,GAAG,MAAM,aAAa,EAAE,CAAA;YAC/B,MAAM,WAAW,CAAC,6BAA6B,CAAC,CAAA;YAChD,YAAY,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC9D,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC9C,CAAC;QAED,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAA;QAErD,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;QAE1C,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;wBACE,SAAS;wBACT,GAAG,QAAQ;qBACZ,EACD,IAAI,EACJ,CAAC,CACF;iBACF;aACF;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC1D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBAC5F;aACF;SACF,CAAA;IACH,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: search_timestamp
|
|
3
|
+
* Find when something specific happens in a video
|
|
4
|
+
*/
|
|
5
|
+
export declare const searchTimestampTool: (params: any) => Promise<{
|
|
6
|
+
content: ({
|
|
7
|
+
type: "text";
|
|
8
|
+
text: string;
|
|
9
|
+
data?: undefined;
|
|
10
|
+
mimeType?: undefined;
|
|
11
|
+
} | {
|
|
12
|
+
type: "image";
|
|
13
|
+
data: string;
|
|
14
|
+
mimeType: string;
|
|
15
|
+
text?: undefined;
|
|
16
|
+
})[];
|
|
17
|
+
isError?: undefined;
|
|
18
|
+
} | {
|
|
19
|
+
isError: boolean;
|
|
20
|
+
content: {
|
|
21
|
+
type: "text";
|
|
22
|
+
text: string;
|
|
23
|
+
}[];
|
|
24
|
+
}>;
|
|
25
|
+
//# sourceMappingURL=searchTimestamp.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"searchTimestamp.d.ts","sourceRoot":"","sources":["../../src/tools/searchTimestamp.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA2CH,eAAO,MAAM,mBAAmB,GAAU,QAAQ,GAAG;;;;;;;;;;;;;;;;;;;EAoIpD,CAAA"}
|