video-context-mcp-server 1.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +380 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +125 -0
- package/dist/index.js.map +1 -0
- package/dist/services/ffmpeg.d.ts +47 -0
- package/dist/services/ffmpeg.d.ts.map +1 -0
- package/dist/services/ffmpeg.js +170 -0
- package/dist/services/ffmpeg.js.map +1 -0
- package/dist/services/geminiClient.d.ts +47 -0
- package/dist/services/geminiClient.d.ts.map +1 -0
- package/dist/services/geminiClient.js +130 -0
- package/dist/services/geminiClient.js.map +1 -0
- package/dist/services/glmClient.d.ts +50 -0
- package/dist/services/glmClient.d.ts.map +1 -0
- package/dist/services/glmClient.js +196 -0
- package/dist/services/glmClient.js.map +1 -0
- package/dist/services/kimiClient.d.ts +45 -0
- package/dist/services/kimiClient.d.ts.map +1 -0
- package/dist/services/kimiClient.js +152 -0
- package/dist/services/kimiClient.js.map +1 -0
- package/dist/services/providerRouter.d.ts +30 -0
- package/dist/services/providerRouter.d.ts.map +1 -0
- package/dist/services/providerRouter.js +62 -0
- package/dist/services/providerRouter.js.map +1 -0
- package/dist/tools/analyzeVideo.d.ts +18 -0
- package/dist/tools/analyzeVideo.d.ts.map +1 -0
- package/dist/tools/analyzeVideo.js +118 -0
- package/dist/tools/analyzeVideo.js.map +1 -0
- package/dist/tools/extractFrames.d.ts +22 -0
- package/dist/tools/extractFrames.d.ts.map +1 -0
- package/dist/tools/extractFrames.js +82 -0
- package/dist/tools/extractFrames.js.map +1 -0
- package/dist/tools/getVideoInfo.d.ts +18 -0
- package/dist/tools/getVideoInfo.d.ts.map +1 -0
- package/dist/tools/getVideoInfo.js +52 -0
- package/dist/tools/getVideoInfo.js.map +1 -0
- package/dist/tools/searchTimestamp.d.ts +25 -0
- package/dist/tools/searchTimestamp.d.ts.map +1 -0
- package/dist/tools/searchTimestamp.js +146 -0
- package/dist/tools/searchTimestamp.js.map +1 -0
- package/dist/tools/summarizeVideo.d.ts +18 -0
- package/dist/tools/summarizeVideo.d.ts.map +1 -0
- package/dist/tools/summarizeVideo.js +159 -0
- package/dist/tools/summarizeVideo.js.map +1 -0
- package/dist/utils/base64.d.ts +35 -0
- package/dist/utils/base64.d.ts.map +1 -0
- package/dist/utils/base64.js +50 -0
- package/dist/utils/base64.js.map +1 -0
- package/dist/utils/logger.d.ts +23 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +34 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/tempFiles.d.ts +20 -0
- package/dist/utils/tempFiles.d.ts.map +1 -0
- package/dist/utils/tempFiles.js +31 -0
- package/dist/utils/tempFiles.js.map +1 -0
- package/dist/utils/videoUtils.d.ts +19 -0
- package/dist/utils/videoUtils.d.ts.map +1 -0
- package/dist/utils/videoUtils.js +38 -0
- package/dist/utils/videoUtils.js.map +1 -0
- package/package.json +62 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: search_timestamp
|
|
3
|
+
* Find when something specific happens in a video
|
|
4
|
+
*/
|
|
5
|
+
import { extractFramesAtInterval, getVideoMetadata, } from '../services/ffmpeg.js';
|
|
6
|
+
import { createGLMClient } from '../services/glmClient.js';
|
|
7
|
+
import { createKimiClient } from '../services/kimiClient.js';
|
|
8
|
+
import { createGeminiClient } from '../services/geminiClient.js';
|
|
9
|
+
import { getDefaultProvider, resolveProviderWithFallback, } from '../services/providerRouter.js';
|
|
10
|
+
import { bufferToBase64String } from '../utils/base64.js';
|
|
11
|
+
import { createTempDir, cleanupTempDir } from '../utils/tempFiles.js';
|
|
12
|
+
import { isRemoteUrl, normalizeVideoPath, downloadVideoToTemp, } from '../utils/videoUtils.js';
|
|
13
|
+
import { logProgress } from '../utils/logger.js';
|
|
14
|
+
function clamp(value, min, max) {
|
|
15
|
+
return Math.max(min, Math.min(max, value));
|
|
16
|
+
}
|
|
17
|
+
function parseLikelyFrameIndex(responseText, maxIndex) {
|
|
18
|
+
const matches = responseText.match(/\d+/g);
|
|
19
|
+
if (!matches || matches.length === 0) {
|
|
20
|
+
return 0;
|
|
21
|
+
}
|
|
22
|
+
for (const match of matches) {
|
|
23
|
+
const value = Number.parseInt(match, 10);
|
|
24
|
+
if (!Number.isNaN(value) && value >= 0 && value <= maxIndex) {
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return clamp(Number.parseInt(matches[0], 10) || 0, 0, maxIndex);
|
|
29
|
+
}
|
|
30
|
+
export const searchTimestampTool = async (params) => {
|
|
31
|
+
let tempDir = null;
|
|
32
|
+
try {
|
|
33
|
+
const { videoPath, query } = params;
|
|
34
|
+
const provider = params.provider ?? getDefaultProvider();
|
|
35
|
+
let resolvedPath;
|
|
36
|
+
if (isRemoteUrl(videoPath)) {
|
|
37
|
+
tempDir = await createTempDir();
|
|
38
|
+
await logProgress('Downloading remote video...');
|
|
39
|
+
resolvedPath = await downloadVideoToTemp(videoPath, tempDir);
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
resolvedPath = normalizeVideoPath(videoPath);
|
|
43
|
+
}
|
|
44
|
+
const metadata = await getVideoMetadata(resolvedPath);
|
|
45
|
+
const glmClient = createGLMClient();
|
|
46
|
+
const kimiClient = createKimiClient();
|
|
47
|
+
const geminiClient = createGeminiClient();
|
|
48
|
+
const selectedProvider = resolveProviderWithFallback(provider, Boolean(kimiClient), Boolean(glmClient), Boolean(geminiClient));
|
|
49
|
+
if (selectedProvider === 'gemini') {
|
|
50
|
+
if (!geminiClient) {
|
|
51
|
+
throw new Error('Gemini client is unavailable. Set GEMINI_API_KEY.');
|
|
52
|
+
}
|
|
53
|
+
await logProgress('Uploading to Gemini...');
|
|
54
|
+
const uploadedVideo = await geminiClient.uploadVideo(resolvedPath);
|
|
55
|
+
await logProgress('Waiting for Gemini to process video...');
|
|
56
|
+
await geminiClient.waitForFileProcessing(uploadedVideo.name);
|
|
57
|
+
await logProgress('Searching timestamp...');
|
|
58
|
+
const modelResponse = await geminiClient.searchTimestamp(uploadedVideo, query);
|
|
59
|
+
if (tempDir)
|
|
60
|
+
await cleanupTempDir(tempDir);
|
|
61
|
+
return {
|
|
62
|
+
content: [
|
|
63
|
+
{
|
|
64
|
+
type: 'text',
|
|
65
|
+
text: modelResponse,
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
type: 'text',
|
|
69
|
+
text: `Provider used: ${selectedProvider}`,
|
|
70
|
+
},
|
|
71
|
+
],
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
await logProgress('Extracting frames...');
|
|
75
|
+
const intervalSec = clamp(Math.ceil(metadata.duration / 20), 2, 15);
|
|
76
|
+
const frames = await extractFramesAtInterval(resolvedPath, intervalSec);
|
|
77
|
+
if (frames.length === 0) {
|
|
78
|
+
throw new Error('No frames were extracted for timestamp search.');
|
|
79
|
+
}
|
|
80
|
+
const images = frames.map((frameBuffer) => ({
|
|
81
|
+
data: bufferToBase64String(frameBuffer),
|
|
82
|
+
mimeType: 'image/png',
|
|
83
|
+
}));
|
|
84
|
+
const searchPrompt = [
|
|
85
|
+
`You are given sequential frames sampled every ${intervalSec} seconds from a video.`,
|
|
86
|
+
`Find which frame index best matches this query: "${query}".`,
|
|
87
|
+
'Return a short answer in this format:',
|
|
88
|
+
'FrameIndex: <number>',
|
|
89
|
+
'Reason: <one sentence>',
|
|
90
|
+
`Valid frame index range: 0 to ${frames.length - 1}.`,
|
|
91
|
+
].join('\n');
|
|
92
|
+
let modelResponse;
|
|
93
|
+
if (selectedProvider === 'glm') {
|
|
94
|
+
if (!glmClient) {
|
|
95
|
+
throw new Error('GLM client is unavailable. Set Z_AI_API_KEY.');
|
|
96
|
+
}
|
|
97
|
+
await logProgress('Searching timestamp...');
|
|
98
|
+
modelResponse = await glmClient.analyzeImages(images, searchPrompt);
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
if (!kimiClient) {
|
|
102
|
+
throw new Error('Kimi client is unavailable. Set MOONSHOT_API_KEY.');
|
|
103
|
+
}
|
|
104
|
+
await logProgress('Searching timestamp...');
|
|
105
|
+
modelResponse = await kimiClient.analyzeImages(images, searchPrompt);
|
|
106
|
+
}
|
|
107
|
+
const frameIndex = parseLikelyFrameIndex(modelResponse, frames.length - 1);
|
|
108
|
+
const timestampSec = frameIndex * intervalSec;
|
|
109
|
+
const startRange = Math.max(0, timestampSec - intervalSec / 2);
|
|
110
|
+
const endRange = Math.min(metadata.duration, timestampSec + intervalSec / 2);
|
|
111
|
+
if (tempDir)
|
|
112
|
+
await cleanupTempDir(tempDir);
|
|
113
|
+
return {
|
|
114
|
+
content: [
|
|
115
|
+
{
|
|
116
|
+
type: 'text',
|
|
117
|
+
text: [
|
|
118
|
+
`Best match for "${query}" appears near ${timestampSec.toFixed(1)}s.`,
|
|
119
|
+
`Estimated range: ${startRange.toFixed(1)}s - ${endRange.toFixed(1)}s.`,
|
|
120
|
+
`Provider used: ${selectedProvider}.`,
|
|
121
|
+
`Model response: ${modelResponse}`,
|
|
122
|
+
].join('\n'),
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
type: 'image',
|
|
126
|
+
data: images[frameIndex].data,
|
|
127
|
+
mimeType: 'image/png',
|
|
128
|
+
},
|
|
129
|
+
],
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
if (tempDir)
|
|
134
|
+
await cleanupTempDir(tempDir).catch(() => { });
|
|
135
|
+
return {
|
|
136
|
+
isError: true,
|
|
137
|
+
content: [
|
|
138
|
+
{
|
|
139
|
+
type: 'text',
|
|
140
|
+
text: `Error searching timestamp: ${error instanceof Error ? error.message : String(error)}`,
|
|
141
|
+
},
|
|
142
|
+
],
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
//# sourceMappingURL=searchTimestamp.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"searchTimestamp.js","sourceRoot":"","sources":["../../src/tools/searchTimestamp.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChE,OAAO,EACL,kBAAkB,EAClB,2BAA2B,GAE5B,MAAM,+BAA+B,CAAA;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACrE,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAEhD,SAAS,KAAK,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAA;AAC5C,CAAC;AAED,SAAS,qBAAqB,CAAC,YAAoB,EAAE,QAAgB;IACnE,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC1C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,CAAA;IACV,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC5D,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;AACjE,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EAAE,MAAW,EAAE,EAAE;IACvD,IAAI,OAAO,GAAkB,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,MAAM,CAAA;QACnC,MAAM,QAAQ,GAAkB,MAAM,CAAC,QAAQ,IAAI,kBAAkB,EAAE,CAAA;QAEvE,IAAI,YAAoB,CAAA;QACxB,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,OAAO,GAAG,MAAM,aAAa,EAAE,CAAA;YAC/B,MAAM,WAAW,CAAC,6BAA6B,CAAC,CAAA;YAChD,YAAY,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;QAC9D,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAA;QAErD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAA;QACnC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAA;QACrC,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAA;QAEzC,MAAM,gBAAgB,GAAkB,2BAA2B,CACjE,QAAQ,EACR,OAAO,CAAC,UAAU,CAAC,EACnB,OAAO,CAAC,SAAS,CAAC,EAClB,OAAO,CAAC,YAAY,CAAC,CACtB,CAAA;QAED,IAAI,gBAAgB,KAAK,QAAQ,EAAE,CAAC;YAClC,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;YACtE,CAAC;YACD,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;YAC3C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;YAClE,MAAM,WAAW,CAAC,wCAAwC,CAAC,CAAA;YAC3D,MAAM,YAAY,CAAC,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;YAC5D,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;YAC3C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,eAAe,CACtD,aAAa,EACb,KAAK,CACN,CAAA;YAED,IAAI,OAAO;gBAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;YAE1C,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,aAAa;qBACpB;oBACD;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,kBAAkB,gBAAgB,EAAE;qBAC3C;iBACF;aACF,CAAA;QACH,CAAC;QAED,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;QACzC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;QACnE,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;QAEvE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,EAAE,oBAAoB,CAAC,WAAW,CAAC;YACvC,QAAQ,EAAE,WAAW;SACtB,CAAC,CAAC,CAAA;QAEH,MAAM,YAAY,GAAG;YACnB,iDAAiD,WAAW,wBAAwB;YACpF,oDAAoD,KAAK,IAAI;YAC7D,uCAAuC;YACvC,sBAAsB;YACtB,wBAAwB;YACxB,iCAAiC,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG;SACtD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEZ,IAAI,aAAqB,CAAA;QACzB,IAAI,gBAAgB,KAAK,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;YACjE,CAAC;YACD,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;YAC3C,aAAa,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;QACrE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;YACtE,CAAC;YACD,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;YAC3C,aAAa,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;QACtE,CAAC;QAED,MAAM,UAAU,GAAG,qBAAqB,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC1E,MAAM,YAAY,GAAG,UAAU,GAAG,WAAW,CAAA;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,WAAW,GAAG,CAAC,CAAC,CAAA;QAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,YAAY,GAAG,WAAW,GAAG,CAAC,CAAC,CAAA;QAE5E,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;QAE1C,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE;wBACJ,mBAAmB,KAAK,kBAAkB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;wBACrE,oBAAoB,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;wBACvE,kBAAkB,gBAAgB,GAAG;wBACrC,mBAAmB,aAAa,EAAE;qBACnC,CAAC,IAAI,CAAC,IAAI,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,OAAgB;oBACtB,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI;oBAC7B,QAAQ,EAAE,WAAW;iBACtB;aACF;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO;YAAE,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC1D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBAC7F;aACF;SACF,CAAA;IACH,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: summarize_video
|
|
3
|
+
* Generate a structured summary of the video
|
|
4
|
+
*/
|
|
5
|
+
export declare const summarizeVideoTool: (params: any) => Promise<{
|
|
6
|
+
content: {
|
|
7
|
+
type: "text";
|
|
8
|
+
text: string;
|
|
9
|
+
}[];
|
|
10
|
+
isError?: undefined;
|
|
11
|
+
} | {
|
|
12
|
+
isError: boolean;
|
|
13
|
+
content: {
|
|
14
|
+
type: "text";
|
|
15
|
+
text: string;
|
|
16
|
+
}[];
|
|
17
|
+
}>;
|
|
18
|
+
//# sourceMappingURL=summarizeVideo.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summarizeVideo.d.ts","sourceRoot":"","sources":["../../src/tools/summarizeVideo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAoCH,eAAO,MAAM,kBAAkB,GAAU,QAAQ,GAAG;;;;;;;;;;;;EA+JnD,CAAA"}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool: summarize_video
|
|
3
|
+
* Generate a structured summary of the video
|
|
4
|
+
*/
|
|
5
|
+
import * as fs from 'fs/promises';
|
|
6
|
+
import { createReadStream } from 'fs';
|
|
7
|
+
import { extractFramesEvenly, getVideoMetadata } from '../services/ffmpeg.js';
|
|
8
|
+
import { createGLMClient } from '../services/glmClient.js';
|
|
9
|
+
import { createKimiClient } from '../services/kimiClient.js';
|
|
10
|
+
import { createGeminiClient } from '../services/geminiClient.js';
|
|
11
|
+
import { getDefaultProvider, getMaxFrames, resolveProviderWithFallback, } from '../services/providerRouter.js';
|
|
12
|
+
import { bufferToBase64String } from '../utils/base64.js';
|
|
13
|
+
import { createTempDir, cleanupTempDir } from '../utils/tempFiles.js';
|
|
14
|
+
import { isRemoteUrl, normalizeVideoPath, downloadVideoToTemp, } from '../utils/videoUtils.js';
|
|
15
|
+
import { logProgress } from '../utils/logger.js';
|
|
16
|
+
const LONG_VIDEO_THRESHOLD_SEC = 5 * 60;
|
|
17
|
+
function buildSummaryPrompt() {
|
|
18
|
+
return [
|
|
19
|
+
'Provide a structured summary of this video.',
|
|
20
|
+
'Format your response with sections:',
|
|
21
|
+
'1) Overview',
|
|
22
|
+
'2) Key scenes',
|
|
23
|
+
'3) Timeline (with approximate timestamps)',
|
|
24
|
+
'4) Notable details',
|
|
25
|
+
].join('\n');
|
|
26
|
+
}
|
|
27
|
+
export const summarizeVideoTool = async (params) => {
|
|
28
|
+
let tempDir = null;
|
|
29
|
+
try {
|
|
30
|
+
const { videoPath } = params;
|
|
31
|
+
const provider = params.provider ?? getDefaultProvider();
|
|
32
|
+
const glmClient = createGLMClient();
|
|
33
|
+
const kimiClient = createKimiClient();
|
|
34
|
+
const geminiClient = createGeminiClient();
|
|
35
|
+
const prompt = buildSummaryPrompt();
|
|
36
|
+
let selectedProvider;
|
|
37
|
+
let summary;
|
|
38
|
+
if (isRemoteUrl(videoPath)) {
|
|
39
|
+
if (provider === 'kimi') {
|
|
40
|
+
throw new Error('Kimi provider does not support direct remote URLs. Use provider=glm, provider=gemini, or a local file.');
|
|
41
|
+
}
|
|
42
|
+
selectedProvider = resolveProviderWithFallback(provider, Boolean(kimiClient), Boolean(glmClient), Boolean(geminiClient));
|
|
43
|
+
if (selectedProvider === 'gemini') {
|
|
44
|
+
if (!geminiClient) {
|
|
45
|
+
throw new Error('Gemini client is unavailable. Set GEMINI_API_KEY.');
|
|
46
|
+
}
|
|
47
|
+
tempDir = await createTempDir();
|
|
48
|
+
await logProgress('Downloading remote video...');
|
|
49
|
+
const localPath = await downloadVideoToTemp(videoPath, tempDir);
|
|
50
|
+
await logProgress('Uploading to Gemini...');
|
|
51
|
+
const uploadedVideo = await geminiClient.uploadVideo(localPath);
|
|
52
|
+
await logProgress('Waiting for Gemini to process video...');
|
|
53
|
+
await geminiClient.waitForFileProcessing(uploadedVideo.name);
|
|
54
|
+
await logProgress('Summarizing video...');
|
|
55
|
+
summary = await geminiClient.analyzeVideo(uploadedVideo, prompt);
|
|
56
|
+
}
|
|
57
|
+
else if (selectedProvider === 'glm') {
|
|
58
|
+
if (!glmClient) {
|
|
59
|
+
throw new Error('GLM provider is required for URL video summarization, but Z_AI_API_KEY is not configured.');
|
|
60
|
+
}
|
|
61
|
+
await logProgress('Summarizing remote video...');
|
|
62
|
+
summary = await glmClient.analyzeVideoUrl(videoPath, prompt);
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
throw new Error('No suitable provider found for remote URL summarization.');
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
const resolvedPath = normalizeVideoPath(videoPath);
|
|
70
|
+
const metadata = await getVideoMetadata(resolvedPath);
|
|
71
|
+
selectedProvider = resolveProviderWithFallback(provider, Boolean(kimiClient), Boolean(glmClient), Boolean(geminiClient));
|
|
72
|
+
if (selectedProvider === 'gemini') {
|
|
73
|
+
if (!geminiClient) {
|
|
74
|
+
throw new Error('Gemini client is unavailable. Set GEMINI_API_KEY.');
|
|
75
|
+
}
|
|
76
|
+
await logProgress('Uploading to Gemini...');
|
|
77
|
+
const uploadedVideo = await geminiClient.uploadVideo(resolvedPath);
|
|
78
|
+
await logProgress('Waiting for Gemini to process video...');
|
|
79
|
+
await geminiClient.waitForFileProcessing(uploadedVideo.name);
|
|
80
|
+
await logProgress('Summarizing video...');
|
|
81
|
+
summary = await geminiClient.analyzeVideo(uploadedVideo, prompt);
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
const isLongVideo = metadata.duration > LONG_VIDEO_THRESHOLD_SEC;
|
|
85
|
+
if (isLongVideo) {
|
|
86
|
+
await logProgress('Extracting keyframes...');
|
|
87
|
+
const frames = await extractFramesEvenly(resolvedPath, getMaxFrames());
|
|
88
|
+
const images = frames.map((frameBuffer) => ({
|
|
89
|
+
data: bufferToBase64String(frameBuffer),
|
|
90
|
+
mimeType: 'image/png',
|
|
91
|
+
}));
|
|
92
|
+
if (selectedProvider === 'glm') {
|
|
93
|
+
if (!glmClient) {
|
|
94
|
+
throw new Error('GLM client is unavailable. Set Z_AI_API_KEY.');
|
|
95
|
+
}
|
|
96
|
+
await logProgress('Summarizing video...');
|
|
97
|
+
summary = await glmClient.analyzeImages(images, prompt);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
if (!kimiClient) {
|
|
101
|
+
throw new Error('Kimi client is unavailable. Set MOONSHOT_API_KEY.');
|
|
102
|
+
}
|
|
103
|
+
await logProgress('Summarizing video...');
|
|
104
|
+
summary = await kimiClient.analyzeImages(images, prompt);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
if (selectedProvider === 'glm') {
|
|
109
|
+
if (!glmClient) {
|
|
110
|
+
throw new Error('GLM client is unavailable. Set Z_AI_API_KEY.');
|
|
111
|
+
}
|
|
112
|
+
await logProgress('Summarizing video...');
|
|
113
|
+
const videoBuffer = await fs.readFile(resolvedPath);
|
|
114
|
+
summary = await glmClient.analyzeVideoBase64(bufferToBase64String(videoBuffer), prompt);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
if (!kimiClient) {
|
|
118
|
+
throw new Error('Kimi client is unavailable. Set MOONSHOT_API_KEY.');
|
|
119
|
+
}
|
|
120
|
+
await logProgress('Uploading to Kimi...');
|
|
121
|
+
const uploadedVideo = await kimiClient.uploadVideo(createReadStream(resolvedPath));
|
|
122
|
+
await logProgress('Summarizing video...');
|
|
123
|
+
summary = await kimiClient.analyzeVideo(uploadedVideo.id, prompt);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (tempDir) {
|
|
129
|
+
await cleanupTempDir(tempDir);
|
|
130
|
+
}
|
|
131
|
+
return {
|
|
132
|
+
content: [
|
|
133
|
+
{
|
|
134
|
+
type: 'text',
|
|
135
|
+
text: summary,
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
type: 'text',
|
|
139
|
+
text: `Provider used: ${selectedProvider}`,
|
|
140
|
+
},
|
|
141
|
+
],
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
if (tempDir) {
|
|
146
|
+
await cleanupTempDir(tempDir).catch(() => { });
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
isError: true,
|
|
150
|
+
content: [
|
|
151
|
+
{
|
|
152
|
+
type: 'text',
|
|
153
|
+
text: `Error summarizing video: ${error instanceof Error ? error.message : String(error)}`,
|
|
154
|
+
},
|
|
155
|
+
],
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
//# sourceMappingURL=summarizeVideo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summarizeVideo.js","sourceRoot":"","sources":["../../src/tools/summarizeVideo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAA;AACrC,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAC7E,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChE,OAAO,EACL,kBAAkB,EAClB,YAAY,EACZ,2BAA2B,GAE5B,MAAM,+BAA+B,CAAA;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACrE,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAEhD,MAAM,wBAAwB,GAAG,CAAC,GAAG,EAAE,CAAA;AAEvC,SAAS,kBAAkB;IACzB,OAAO;QACL,6CAA6C;QAC7C,qCAAqC;QACrC,aAAa;QACb,eAAe;QACf,2CAA2C;QAC3C,oBAAoB;KACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACd,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EAAE,MAAW,EAAE,EAAE;IACtD,IAAI,OAAO,GAAkB,IAAI,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAA;QAC5B,MAAM,QAAQ,GAAkB,MAAM,CAAC,QAAQ,IAAI,kBAAkB,EAAE,CAAA;QAEvE,MAAM,SAAS,GAAG,eAAe,EAAE,CAAA;QACnC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAA;QACrC,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAA;QAEzC,MAAM,MAAM,GAAG,kBAAkB,EAAE,CAAA;QACnC,IAAI,gBAA+B,CAAA;QACnC,IAAI,OAAe,CAAA;QAEnB,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACb,wGAAwG,CACzG,CAAA;YACH,CAAC;YAED,gBAAgB,GAAG,2BAA2B,CAC5C,QAAQ,EACR,OAAO,CAAC,UAAU,CAAC,EACnB,OAAO,CAAC,SAAS,CAAC,EAClB,OAAO,CAAC,YAAY,CAAC,CACtB,CAAA;YAED,IAAI,gBAAgB,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;gBACtE,CAAC;gBACD,OAAO,GAAG,MAAM,aAAa,EAAE,CAAA;gBAC/B,MAAM,WAAW,CAAC,6BAA6B,CAAC,CAAA;gBAChD,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;gBAC/D,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;gBAC3C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;gBAC/D,MAAM,WAAW,CAAC,wCAAwC,CAAC,CAAA;gBAC3D,MAAM,YAAY,CAAC,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;gBAC5D,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;gBACzC,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;YAClE,CAAC;iBAAM,IAAI,gBAAgB,KAAK,KAAK,EAAE,CAAC;gBACtC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CACb,2FAA2F,CAC5F,CAAA;gBACH,CAAC;gBACD,MAAM,WAAW,CAAC,6BAA6B,CAAC,CAAA;gBAChD,OAAO,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;YAC9D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,0DAA0D,CAC3D,CAAA;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;YAClD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAA;YAErD,gBAAgB,GAAG,2BAA2B,CAC5C,QAAQ,EACR,OAAO,CAAC,UAAU,CAAC,EACnB,OAAO,CAAC,SAAS,CAAC,EAClB,OAAO,CAAC,YAAY,CAAC,CACtB,CAAA;YAED,IAAI,gBAAgB,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAA;gBACtE,CAAC;gBACD,MAAM,WAAW,CAAC,wBAAwB,CAAC,CAAA;gBAC3C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;gBAClE,MAAM,WAAW,CAAC,wCAAwC,CAAC,CAAA;gBAC3D,MAAM,YAAY,CAAC,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;gBAC5D,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;gBACzC,OAAO,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;YAClE,CAAC;iBAAM,CAAC;gBACN,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,GAAG,wBAAwB,CAAA;gBAEhE,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,WAAW,CAAC,yBAAyB,CAAC,CAAA;oBAC5C,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC,CAAA;oBACtE,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;wBAC1C,IAAI,EAAE,oBAAoB,CAAC,WAAW,CAAC;wBACvC,QAAQ,EAAE,WAAW;qBACtB,CAAC,CAAC,CAAA;oBAEH,IAAI,gBAAgB,KAAK,KAAK,EAAE,CAAC;wBAC/B,IAAI,CAAC,SAAS,EAAE,CAAC;4BACf,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;wBACjE,CAAC;wBACD,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;wBACzC,OAAO,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;oBACzD,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,UAAU,EAAE,CAAC;4BAChB,MAAM,IAAI,KAAK,CACb,mDAAmD,CACpD,CAAA;wBACH,CAAC;wBACD,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;wBACzC,OAAO,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;oBAC1D,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,gBAAgB,KAAK,KAAK,EAAE,CAAC;wBAC/B,IAAI,CAAC,SAAS,EAAE,CAAC;4BACf,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;wBACjE,CAAC;wBACD,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;wBACzC,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;wBACnD,OAAO,GAAG,MAAM,SAAS,CAAC,kBAAkB,CAC1C,oBAAoB,CAAC,WAAW,CAAC,EACjC,MAAM,CACP,CAAA;oBACH,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,UAAU,EAAE,CAAC;4BAChB,MAAM,IAAI,KAAK,CACb,mDAAmD,CACpD,CAAA;wBACH,CAAC;wBACD,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;wBACzC,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,WAAW,CAChD,gBAAgB,CAAC,YAAY,CAAC,CAC/B,CAAA;wBACD,MAAM,WAAW,CAAC,sBAAsB,CAAC,CAAA;wBACzC,OAAO,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;oBACnE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;QAC/B,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,kBAAkB,gBAAgB,EAAE;iBAC3C;aACF;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC/C,CAAC;QACD,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBAC3F;aACF;SACF,CAAA;IACH,CAAC;AACH,CAAC,CAAA"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Base64 encoding utilities for images and videos
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Convert a Buffer to a base64 data URI
|
|
6
|
+
* @param buffer - The data buffer
|
|
7
|
+
* @param mimeType - MIME type (e.g., "image/png", "video/mp4")
|
|
8
|
+
* @returns Base64 data URI string
|
|
9
|
+
*/
|
|
10
|
+
export declare function bufferToBase64(buffer: Buffer, mimeType: string): string;
|
|
11
|
+
/**
|
|
12
|
+
* Convert a Buffer to base64 without the data URI prefix
|
|
13
|
+
* @param buffer - The data buffer
|
|
14
|
+
* @returns Base64 string
|
|
15
|
+
*/
|
|
16
|
+
export declare function bufferToBase64String(buffer: Buffer): string;
|
|
17
|
+
/**
|
|
18
|
+
* Extract the MIME type from a data URI
|
|
19
|
+
* @param dataUri - Data URI string (e.g., "data:image/png;base64,abc123...")
|
|
20
|
+
* @returns MIME type (e.g., "image/png")
|
|
21
|
+
*/
|
|
22
|
+
export declare function getMimeTypeFromDataUri(dataUri: string): string;
|
|
23
|
+
/**
|
|
24
|
+
* Extract the base64 content from a data URI
|
|
25
|
+
* @param dataUri - Data URI string (e.g., "data:image/png;base64,abc123...")
|
|
26
|
+
* @returns Base64 string without prefix
|
|
27
|
+
*/
|
|
28
|
+
export declare function getBase64FromDataUri(dataUri: string): string;
|
|
29
|
+
/**
|
|
30
|
+
* Convert base64 string to Buffer
|
|
31
|
+
* @param base64 - Base64 string (with or without data URI prefix)
|
|
32
|
+
* @returns Buffer
|
|
33
|
+
*/
|
|
34
|
+
export declare function base64ToBuffer(base64: string): Buffer;
|
|
35
|
+
//# sourceMappingURL=base64.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base64.d.ts","sourceRoot":"","sources":["../../src/utils/base64.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGvE;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAG9D;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAG5D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAIrD"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Base64 encoding utilities for images and videos
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Convert a Buffer to a base64 data URI
|
|
6
|
+
* @param buffer - The data buffer
|
|
7
|
+
* @param mimeType - MIME type (e.g., "image/png", "video/mp4")
|
|
8
|
+
* @returns Base64 data URI string
|
|
9
|
+
*/
|
|
10
|
+
export function bufferToBase64(buffer, mimeType) {
|
|
11
|
+
const base64 = buffer.toString('base64');
|
|
12
|
+
return `data:${mimeType};base64,${base64}`;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Convert a Buffer to base64 without the data URI prefix
|
|
16
|
+
* @param buffer - The data buffer
|
|
17
|
+
* @returns Base64 string
|
|
18
|
+
*/
|
|
19
|
+
export function bufferToBase64String(buffer) {
|
|
20
|
+
return buffer.toString('base64');
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Extract the MIME type from a data URI
|
|
24
|
+
* @param dataUri - Data URI string (e.g., "data:image/png;base64,abc123...")
|
|
25
|
+
* @returns MIME type (e.g., "image/png")
|
|
26
|
+
*/
|
|
27
|
+
export function getMimeTypeFromDataUri(dataUri) {
|
|
28
|
+
const match = dataUri.match(/^data:([^;]+);base64,/);
|
|
29
|
+
return match ? match[1] : 'application/octet-stream';
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Extract the base64 content from a data URI
|
|
33
|
+
* @param dataUri - Data URI string (e.g., "data:image/png;base64,abc123...")
|
|
34
|
+
* @returns Base64 string without prefix
|
|
35
|
+
*/
|
|
36
|
+
export function getBase64FromDataUri(dataUri) {
|
|
37
|
+
const match = dataUri.match(/^data:[^;]+;base64,(.+)$/);
|
|
38
|
+
return match ? match[1] : dataUri;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Convert base64 string to Buffer
|
|
42
|
+
* @param base64 - Base64 string (with or without data URI prefix)
|
|
43
|
+
* @returns Buffer
|
|
44
|
+
*/
|
|
45
|
+
export function base64ToBuffer(base64) {
|
|
46
|
+
// Remove data URI prefix if present
|
|
47
|
+
const cleanBase64 = base64.includes(',') ? base64.split(',')[1] : base64;
|
|
48
|
+
return Buffer.from(cleanBase64, 'base64');
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=base64.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base64.js","sourceRoot":"","sources":["../../src/utils/base64.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB;IAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACxC,OAAO,QAAQ,QAAQ,WAAW,MAAM,EAAE,CAAA;AAC5C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;AAClC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAAe;IACpD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAA;IACpD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAA;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;IACvD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;AACnC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,oCAAoC;IACpC,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IACxE,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;AAC3C,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Logger utility for emitting MCP progress notifications from tool handlers.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* // In index.ts after creating the server:
|
|
6
|
+
* setLoggerServer(server)
|
|
7
|
+
*
|
|
8
|
+
* // In tool handlers:
|
|
9
|
+
* await logProgress('Uploading to Gemini...')
|
|
10
|
+
*/
|
|
11
|
+
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
12
|
+
/**
|
|
13
|
+
* Register the McpServer instance so that logProgress can use it.
|
|
14
|
+
* Call this once from index.ts after creating the server.
|
|
15
|
+
*/
|
|
16
|
+
export declare function setLoggerServer(server: McpServer): void;
|
|
17
|
+
/**
|
|
18
|
+
* Emit an info-level progress log via the MCP logging notification.
|
|
19
|
+
* Silently swallows errors so that logging failures never break a tool call.
|
|
20
|
+
* @param message - Human-readable progress message
|
|
21
|
+
*/
|
|
22
|
+
export declare function logProgress(message: string): Promise<void>;
|
|
23
|
+
//# sourceMappingURL=logger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AAIxE;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOhE"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Logger utility for emitting MCP progress notifications from tool handlers.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* // In index.ts after creating the server:
|
|
6
|
+
* setLoggerServer(server)
|
|
7
|
+
*
|
|
8
|
+
* // In tool handlers:
|
|
9
|
+
* await logProgress('Uploading to Gemini...')
|
|
10
|
+
*/
|
|
11
|
+
let _server = null;
|
|
12
|
+
/**
|
|
13
|
+
* Register the McpServer instance so that logProgress can use it.
|
|
14
|
+
* Call this once from index.ts after creating the server.
|
|
15
|
+
*/
|
|
16
|
+
export function setLoggerServer(server) {
|
|
17
|
+
_server = server;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Emit an info-level progress log via the MCP logging notification.
|
|
21
|
+
* Silently swallows errors so that logging failures never break a tool call.
|
|
22
|
+
* @param message - Human-readable progress message
|
|
23
|
+
*/
|
|
24
|
+
export async function logProgress(message) {
|
|
25
|
+
if (!_server)
|
|
26
|
+
return;
|
|
27
|
+
try {
|
|
28
|
+
await _server.sendLoggingMessage({ level: 'info', data: message });
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
// Swallow logging errors – never break a tool call because of logging
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,IAAI,OAAO,GAAqB,IAAI,CAAA;AAEpC;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,OAAO,GAAG,MAAM,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAe;IAC/C,IAAI,CAAC,OAAO;QAAE,OAAM;IACpB,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAA;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,sEAAsE;IACxE,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Temp directory manager for frame extraction operations
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Create a temporary directory for video processing
|
|
6
|
+
* @returns Path to the created temp directory
|
|
7
|
+
*/
|
|
8
|
+
export declare function createTempDir(): Promise<string>;
|
|
9
|
+
/**
|
|
10
|
+
* Clean up a temporary directory
|
|
11
|
+
* @param dirPath - Path to the temp directory to clean up
|
|
12
|
+
*/
|
|
13
|
+
export declare function cleanupTempDir(dirPath: string): Promise<void>;
|
|
14
|
+
/**
|
|
15
|
+
* Generate a unique temp filename
|
|
16
|
+
* @param extension - File extension (e.g., ".png")
|
|
17
|
+
* @returns Unique filename
|
|
18
|
+
*/
|
|
19
|
+
export declare function generateTempFilename(extension: string): string;
|
|
20
|
+
//# sourceMappingURL=tempFiles.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tempFiles.d.ts","sourceRoot":"","sources":["../../src/utils/tempFiles.ts"],"names":[],"mappings":"AAIA;;GAEG;AAEH;;;GAGG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAErD;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEnE;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAI9D"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import * as path from 'path';
|
|
2
|
+
import { mkdtemp, rm } from 'fs/promises';
|
|
3
|
+
import { tmpdir } from 'os';
|
|
4
|
+
/**
|
|
5
|
+
* Temp directory manager for frame extraction operations
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Create a temporary directory for video processing
|
|
9
|
+
* @returns Path to the created temp directory
|
|
10
|
+
*/
|
|
11
|
+
export async function createTempDir() {
|
|
12
|
+
return await mkdtemp(path.join(tmpdir(), 'video-mcp-'));
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Clean up a temporary directory
|
|
16
|
+
* @param dirPath - Path to the temp directory to clean up
|
|
17
|
+
*/
|
|
18
|
+
export async function cleanupTempDir(dirPath) {
|
|
19
|
+
await rm(dirPath, { recursive: true, force: true });
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Generate a unique temp filename
|
|
23
|
+
* @param extension - File extension (e.g., ".png")
|
|
24
|
+
* @returns Unique filename
|
|
25
|
+
*/
|
|
26
|
+
export function generateTempFilename(extension) {
|
|
27
|
+
const timestamp = Date.now();
|
|
28
|
+
const random = Math.random().toString(36).substring(2, 9);
|
|
29
|
+
return `${timestamp}-${random}${extension}`;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=tempFiles.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tempFiles.js","sourceRoot":"","sources":["../../src/utils/tempFiles.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAA;AAE3B;;GAEG;AAEH;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAA;AACzD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAe;IAClD,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AACrD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IACpD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IACzD,OAAO,GAAG,SAAS,IAAI,MAAM,GAAG,SAAS,EAAE,CAAA;AAC7C,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared video utilities for path normalization, URL detection, and video downloading
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Returns true if the value is an http or https URL
|
|
6
|
+
*/
|
|
7
|
+
export declare function isRemoteUrl(value: string): boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Normalizes a video path, stripping file:// URIs to local filesystem paths.
|
|
10
|
+
* Non-file:// paths are returned as-is.
|
|
11
|
+
*/
|
|
12
|
+
export declare function normalizeVideoPath(videoPath: string): string;
|
|
13
|
+
/**
|
|
14
|
+
* Downloads a remote video URL to a temp directory and returns the local path.
|
|
15
|
+
* @param url - Remote http(s) URL
|
|
16
|
+
* @param tempDir - Directory to download into
|
|
17
|
+
*/
|
|
18
|
+
export declare function downloadVideoToTemp(url: string, tempDir: string): Promise<string>;
|
|
19
|
+
//# sourceMappingURL=videoUtils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"videoUtils.d.ts","sourceRoot":"","sources":["../../src/utils/videoUtils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH;;GAEG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAElD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAK5D;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CACvC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAQjB"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared video utilities for path normalization, URL detection, and video downloading
|
|
3
|
+
*/
|
|
4
|
+
import { createWriteStream } from 'fs';
|
|
5
|
+
import * as path from 'path';
|
|
6
|
+
import { pipeline } from 'stream/promises';
|
|
7
|
+
import { generateTempFilename } from './tempFiles.js';
|
|
8
|
+
/**
|
|
9
|
+
* Returns true if the value is an http or https URL
|
|
10
|
+
*/
|
|
11
|
+
export function isRemoteUrl(value) {
|
|
12
|
+
return /^https?:\/\//i.test(value);
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Normalizes a video path, stripping file:// URIs to local filesystem paths.
|
|
16
|
+
* Non-file:// paths are returned as-is.
|
|
17
|
+
*/
|
|
18
|
+
export function normalizeVideoPath(videoPath) {
|
|
19
|
+
if (videoPath.startsWith('file://')) {
|
|
20
|
+
return decodeURIComponent(videoPath.replace(/^file:\/\//i, ''));
|
|
21
|
+
}
|
|
22
|
+
return videoPath;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Downloads a remote video URL to a temp directory and returns the local path.
|
|
26
|
+
* @param url - Remote http(s) URL
|
|
27
|
+
* @param tempDir - Directory to download into
|
|
28
|
+
*/
|
|
29
|
+
export async function downloadVideoToTemp(url, tempDir) {
|
|
30
|
+
const response = await fetch(url);
|
|
31
|
+
if (!response.ok) {
|
|
32
|
+
throw new Error(`Failed to download video: ${response.statusText}`);
|
|
33
|
+
}
|
|
34
|
+
const tempFilePath = path.join(tempDir, generateTempFilename('.mp4'));
|
|
35
|
+
await pipeline(response.body, createWriteStream(tempFilePath));
|
|
36
|
+
return tempFilePath;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=videoUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"videoUtils.js","sourceRoot":"","sources":["../../src/utils/videoUtils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAA;AACtC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAErD;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,OAAO,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,OAAO,kBAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,CAAA;IACjE,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,GAAW,EACX,OAAe;IAEf,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;IACjC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IACrE,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAA;IACrE,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAW,EAAE,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAA;IACrE,OAAO,YAAY,CAAA;AACrB,CAAC"}
|