video-context-mcp-server 1.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +380 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +125 -0
- package/dist/index.js.map +1 -0
- package/dist/services/ffmpeg.d.ts +47 -0
- package/dist/services/ffmpeg.d.ts.map +1 -0
- package/dist/services/ffmpeg.js +170 -0
- package/dist/services/ffmpeg.js.map +1 -0
- package/dist/services/geminiClient.d.ts +47 -0
- package/dist/services/geminiClient.d.ts.map +1 -0
- package/dist/services/geminiClient.js +130 -0
- package/dist/services/geminiClient.js.map +1 -0
- package/dist/services/glmClient.d.ts +50 -0
- package/dist/services/glmClient.d.ts.map +1 -0
- package/dist/services/glmClient.js +196 -0
- package/dist/services/glmClient.js.map +1 -0
- package/dist/services/kimiClient.d.ts +45 -0
- package/dist/services/kimiClient.d.ts.map +1 -0
- package/dist/services/kimiClient.js +152 -0
- package/dist/services/kimiClient.js.map +1 -0
- package/dist/services/providerRouter.d.ts +30 -0
- package/dist/services/providerRouter.d.ts.map +1 -0
- package/dist/services/providerRouter.js +62 -0
- package/dist/services/providerRouter.js.map +1 -0
- package/dist/tools/analyzeVideo.d.ts +18 -0
- package/dist/tools/analyzeVideo.d.ts.map +1 -0
- package/dist/tools/analyzeVideo.js +118 -0
- package/dist/tools/analyzeVideo.js.map +1 -0
- package/dist/tools/extractFrames.d.ts +22 -0
- package/dist/tools/extractFrames.d.ts.map +1 -0
- package/dist/tools/extractFrames.js +82 -0
- package/dist/tools/extractFrames.js.map +1 -0
- package/dist/tools/getVideoInfo.d.ts +18 -0
- package/dist/tools/getVideoInfo.d.ts.map +1 -0
- package/dist/tools/getVideoInfo.js +52 -0
- package/dist/tools/getVideoInfo.js.map +1 -0
- package/dist/tools/searchTimestamp.d.ts +25 -0
- package/dist/tools/searchTimestamp.d.ts.map +1 -0
- package/dist/tools/searchTimestamp.js +146 -0
- package/dist/tools/searchTimestamp.js.map +1 -0
- package/dist/tools/summarizeVideo.d.ts +18 -0
- package/dist/tools/summarizeVideo.d.ts.map +1 -0
- package/dist/tools/summarizeVideo.js +159 -0
- package/dist/tools/summarizeVideo.js.map +1 -0
- package/dist/utils/base64.d.ts +35 -0
- package/dist/utils/base64.d.ts.map +1 -0
- package/dist/utils/base64.js +50 -0
- package/dist/utils/base64.js.map +1 -0
- package/dist/utils/logger.d.ts +23 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +34 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/tempFiles.d.ts +20 -0
- package/dist/utils/tempFiles.d.ts.map +1 -0
- package/dist/utils/tempFiles.js +31 -0
- package/dist/utils/tempFiles.js.map +1 -0
- package/dist/utils/videoUtils.d.ts +19 -0
- package/dist/utils/videoUtils.d.ts.map +1 -0
- package/dist/utils/videoUtils.js +38 -0
- package/dist/utils/videoUtils.js.map +1 -0
- package/package.json +62 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import ffmpeg from 'fluent-ffmpeg';
|
|
2
|
+
import ffmpegInstaller from '@ffmpeg-installer/ffmpeg';
|
|
3
|
+
import ffprobeInstaller from '@ffprobe-installer/ffprobe';
|
|
4
|
+
import * as fs from 'fs/promises';
|
|
5
|
+
import * as path from 'path';
|
|
6
|
+
import { mkdtemp, rm } from 'fs/promises';
|
|
7
|
+
import { tmpdir } from 'os';
|
|
8
|
+
// Set paths to bundled ffmpeg and ffprobe binaries
|
|
9
|
+
ffmpeg.setFfmpegPath(ffmpegInstaller.path);
|
|
10
|
+
ffmpeg.setFfprobePath(ffprobeInstaller.path);
|
|
11
|
+
/**
|
|
12
|
+
* Get video metadata using ffprobe
|
|
13
|
+
* @param videoPath - Path to the video file
|
|
14
|
+
* @returns Video metadata
|
|
15
|
+
*/
|
|
16
|
+
export async function getVideoMetadata(videoPath) {
|
|
17
|
+
return new Promise((resolve, reject) => {
|
|
18
|
+
ffmpeg.ffprobe(videoPath, (err, metadata) => {
|
|
19
|
+
if (err) {
|
|
20
|
+
reject(err);
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
const videoStream = metadata.streams.find((s) => s.codec_type === 'video');
|
|
24
|
+
if (!videoStream) {
|
|
25
|
+
reject(new Error('No video stream found in file'));
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
resolve({
|
|
29
|
+
duration: metadata.format.duration || 0,
|
|
30
|
+
width: videoStream.width || 0,
|
|
31
|
+
height: videoStream.height || 0,
|
|
32
|
+
fps: parseFps(videoStream.r_frame_rate),
|
|
33
|
+
codec: videoStream.codec_name || 'unknown',
|
|
34
|
+
fileSize: metadata.format.size || 0,
|
|
35
|
+
format: metadata.format.format_name || 'unknown',
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Extract a single frame at a specific timestamp
|
|
42
|
+
* @param videoPath - Path to the video file
|
|
43
|
+
* @param timestampSec - Timestamp in seconds
|
|
44
|
+
* @returns Buffer containing the PNG frame
|
|
45
|
+
*/
|
|
46
|
+
export async function extractFrameAt(videoPath, timestampSec) {
|
|
47
|
+
const tempDir = await mkdtemp(path.join(tmpdir(), 'video-mcp-'));
|
|
48
|
+
const outputPath = path.join(tempDir, 'frame.png');
|
|
49
|
+
try {
|
|
50
|
+
await new Promise((resolve, reject) => {
|
|
51
|
+
ffmpeg(videoPath)
|
|
52
|
+
.seekInput(timestampSec)
|
|
53
|
+
.frames(1)
|
|
54
|
+
.output(outputPath)
|
|
55
|
+
.on('end', () => resolve())
|
|
56
|
+
.on('error', reject)
|
|
57
|
+
.run();
|
|
58
|
+
});
|
|
59
|
+
const frameBuffer = await fs.readFile(outputPath);
|
|
60
|
+
return frameBuffer;
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Extract N evenly-spaced frames from a video
|
|
68
|
+
* @param videoPath - Path to the video file
|
|
69
|
+
* @param count - Number of frames to extract
|
|
70
|
+
* @returns Array of PNG buffers
|
|
71
|
+
*/
|
|
72
|
+
export async function extractFramesEvenly(videoPath, count) {
|
|
73
|
+
const tempDir = await mkdtemp(path.join(tmpdir(), 'video-mcp-'));
|
|
74
|
+
try {
|
|
75
|
+
await new Promise((resolve, reject) => {
|
|
76
|
+
ffmpeg(videoPath)
|
|
77
|
+
.screenshots({
|
|
78
|
+
count,
|
|
79
|
+
folder: tempDir,
|
|
80
|
+
filename: 'frame-%i.png',
|
|
81
|
+
size: '1280x?', // Resize width to 1280, maintain aspect ratio
|
|
82
|
+
})
|
|
83
|
+
.on('end', () => resolve())
|
|
84
|
+
.on('error', reject);
|
|
85
|
+
});
|
|
86
|
+
// Read all generated frame files
|
|
87
|
+
const framePaths = (await fs.readdir(tempDir))
|
|
88
|
+
.filter((f) => f.endsWith('.png'))
|
|
89
|
+
.sort();
|
|
90
|
+
const frames = await Promise.all(framePaths.map((f) => fs.readFile(path.join(tempDir, f))));
|
|
91
|
+
return frames;
|
|
92
|
+
}
|
|
93
|
+
finally {
|
|
94
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Extract frames at regular intervals
|
|
99
|
+
* @param videoPath - Path to the video file
|
|
100
|
+
* @param intervalSec - Interval in seconds between frames
|
|
101
|
+
* @returns Array of PNG buffers
|
|
102
|
+
*/
|
|
103
|
+
export async function extractFramesAtInterval(videoPath, intervalSec) {
|
|
104
|
+
const tempDir = await mkdtemp(path.join(tmpdir(), 'video-mcp-'));
|
|
105
|
+
try {
|
|
106
|
+
await new Promise((resolve, reject) => {
|
|
107
|
+
ffmpeg(videoPath)
|
|
108
|
+
.outputOptions([`-vf fps=1/${intervalSec}`])
|
|
109
|
+
.output(path.join(tempDir, 'frame-%04d.png'))
|
|
110
|
+
.on('end', () => resolve())
|
|
111
|
+
.on('error', reject)
|
|
112
|
+
.run();
|
|
113
|
+
});
|
|
114
|
+
const framePaths = (await fs.readdir(tempDir))
|
|
115
|
+
.filter((f) => f.endsWith('.png'))
|
|
116
|
+
.sort();
|
|
117
|
+
const frames = await Promise.all(framePaths.map((f) => fs.readFile(path.join(tempDir, f))));
|
|
118
|
+
return frames;
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Extract frames at specific timestamps
|
|
126
|
+
* @param videoPath - Path to the video file
|
|
127
|
+
* @param timestamps - Array of timestamps in seconds
|
|
128
|
+
* @returns Array of PNG buffers (same order as timestamps)
|
|
129
|
+
*/
|
|
130
|
+
export async function extractFramesAtTimestamps(videoPath, timestamps) {
|
|
131
|
+
const frames = [];
|
|
132
|
+
const tempDir = await mkdtemp(path.join(tmpdir(), 'video-mcp-'));
|
|
133
|
+
try {
|
|
134
|
+
for (let i = 0; i < timestamps.length; i++) {
|
|
135
|
+
const outputPath = path.join(tempDir, `frame-${i}.png`);
|
|
136
|
+
const timestamp = timestamps[i];
|
|
137
|
+
await new Promise((resolve, reject) => {
|
|
138
|
+
ffmpeg(videoPath)
|
|
139
|
+
.seekInput(timestamp)
|
|
140
|
+
.frames(1)
|
|
141
|
+
.output(outputPath)
|
|
142
|
+
.on('end', () => resolve())
|
|
143
|
+
.on('error', reject)
|
|
144
|
+
.run();
|
|
145
|
+
});
|
|
146
|
+
frames.push(await fs.readFile(outputPath));
|
|
147
|
+
}
|
|
148
|
+
return frames;
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Parse fps string (e.g., "24/1" or "30/1") to number
|
|
156
|
+
*/
|
|
157
|
+
function parseFps(fpsString) {
|
|
158
|
+
if (!fpsString)
|
|
159
|
+
return 0;
|
|
160
|
+
const parts = fpsString.split('/');
|
|
161
|
+
if (parts.length === 2) {
|
|
162
|
+
const numerator = parseInt(parts[0], 10);
|
|
163
|
+
const denominator = parseInt(parts[1], 10);
|
|
164
|
+
if (denominator !== 0) {
|
|
165
|
+
return numerator / denominator;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return parseInt(fpsString, 10) || 0;
|
|
169
|
+
}
|
|
170
|
+
//# sourceMappingURL=ffmpeg.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ffmpeg.js","sourceRoot":"","sources":["../../src/services/ffmpeg.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,eAAe,CAAA;AAClC,OAAO,eAAe,MAAM,0BAA0B,CAAA;AACtD,OAAO,gBAAgB,MAAM,4BAA4B,CAAA;AACzD,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAA;AAE3B,mDAAmD;AACnD,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;AAC1C,MAAM,CAAC,cAAc,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;AAe5C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAiB;IAEjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE;YAC1C,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,CAAC,GAAG,CAAC,CAAA;gBACX,OAAM;YACR,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,OAAO,CAAC,CAAA;YAE1E,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC,CAAA;gBAClD,OAAM;YACR,CAAC;YAED,OAAO,CAAC;gBACN,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC;gBACvC,KAAK,EAAE,WAAW,CAAC,KAAK,IAAI,CAAC;gBAC7B,MAAM,EAAE,WAAW,CAAC,MAAM,IAAI,CAAC;gBAC/B,GAAG,EAAE,QAAQ,CAAC,WAAW,CAAC,YAAY,CAAC;gBACvC,KAAK,EAAE,WAAW,CAAC,UAAU,IAAI,SAAS;gBAC1C,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;gBACnC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,IAAI,SAAS;aACjD,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,YAAoB;IAEpB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAA;IAChE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;IAElD,IAAI,CAAC;QACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,SAAS,CAAC;iBACd,SAAS,CAAC,YAAY,CAAC;iBACvB,MAAM,CAAC,CAAC,CAAC;iBACT,MAAM,CAAC,UAAU,CAAC;iBAClB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;iBAC1B,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;iBACnB,GAAG,EAAE,CAAA;QACV,CAAC,CAAC,CAAA;QAEF,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;QACjD,OAAO,WAAW,CAAA;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IACrD,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,SAAiB,EACjB,KAAa;IAEb,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAA;IAEhE,IAAI,CAAC;QACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,SAAS,CAAC;iBACd,WAAW,CAAC;gBACX,KAAK;gBACL,MAAM,EAAE,OAAO;gBACf,QAAQ,EAAE,cAAc;gBACxB,IAAI,EAAE,QAAQ,EAAE,8CAA8C;aAC/D,CAAC;iBACD,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;iBAC1B,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QACxB,CAAC,CAAC,CAAA;QAEF,iCAAiC;QACjC,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;aAC3C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;aACjC,IAAI,EAAE,CAAA;QAET,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAC1D,CAAA;QAED,OAAO,MAAM,CAAA;IACf,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IACrD,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,SAAiB,EACjB,WAAmB;IAEnB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAA;IAEhE,IAAI,CAAC;QACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,SAAS,CAAC;iBACd,aAAa,CAAC,CAAC,aAAa,WAAW,EAAE,CAAC,CAAC;iBAC3C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;iBAC5C,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;iBAC1B,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;iBACnB,GAAG,EAAE,CAAA;QACV,CAAC,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;aAC3C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;aACjC,IAAI,EAAE,CAAA;QAET,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAC1D,CAAA;QAED,OAAO,MAAM,CAAA;IACf,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IACrD,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAiB,EACjB,UAAoB;IAEpB,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAA;IAEhE,IAAI,CAAC;QACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAA;YACvD,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAE/B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,MAAM,CAAC,SAAS,CAAC;qBACd,SAAS,CAAC,SAAS,CAAC;qBACpB,MAAM,CAAC,CAAC,CAAC;qBACT,MAAM,CAAC,UAAU,CAAC;qBAClB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;qBAC1B,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;qBACnB,GAAG,EAAE,CAAA;YACV,CAAC,CAAC,CAAA;YAEF,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAA;QAC5C,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IACrD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,SAA6B;IAC7C,IAAI,CAAC,SAAS;QAAE,OAAO,CAAC,CAAA;IAExB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC1C,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,SAAS,GAAG,WAAW,CAAA;QAChC,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;AACrC,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export interface GeminiClientOptions {
|
|
2
|
+
apiKey: string;
|
|
3
|
+
}
|
|
4
|
+
export declare class GeminiClient {
|
|
5
|
+
private client;
|
|
6
|
+
constructor(options: GeminiClientOptions);
|
|
7
|
+
/**
|
|
8
|
+
* Upload a video file to Gemini's storage
|
|
9
|
+
* @param filePath - The local path to the video file
|
|
10
|
+
* @returns File object for use in prompts
|
|
11
|
+
*/
|
|
12
|
+
uploadVideo(filePath: string): Promise<any>;
|
|
13
|
+
/**
|
|
14
|
+
* Wait for a file to finish processing
|
|
15
|
+
* @param name - The file name returned from uploadVideo()
|
|
16
|
+
*/
|
|
17
|
+
waitForFileProcessing(name: string): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Analyze a video using the uploaded file object
|
|
20
|
+
* @param fileObject - The file object returned from uploadVideo()
|
|
21
|
+
* @param question - The question to ask about the video
|
|
22
|
+
* @returns The AI analysis response
|
|
23
|
+
*/
|
|
24
|
+
analyzeVideo(fileObject: any, question: string): Promise<string>;
|
|
25
|
+
/**
|
|
26
|
+
* Summarize a video using the uploaded file object
|
|
27
|
+
* @param fileObject - The file object returned from uploadVideo()
|
|
28
|
+
* @returns The AI summary response
|
|
29
|
+
*/
|
|
30
|
+
summarizeVideo(fileObject: any): Promise<string>;
|
|
31
|
+
/**
|
|
32
|
+
* Search for a timestamp in a video using the uploaded file object
|
|
33
|
+
* @param fileObject - The file object returned from uploadVideo()
|
|
34
|
+
* @param query - The event or object to search for
|
|
35
|
+
* @returns The AI search response
|
|
36
|
+
*/
|
|
37
|
+
searchTimestamp(fileObject: any, query: string): Promise<string>;
|
|
38
|
+
/**
|
|
39
|
+
* Analyze multiple images (e.g., extracted frames)
|
|
40
|
+
*/
|
|
41
|
+
analyzeImages(images: Array<{
|
|
42
|
+
data: string;
|
|
43
|
+
mimeType: string;
|
|
44
|
+
}>, prompt: string): Promise<string>;
|
|
45
|
+
}
|
|
46
|
+
export declare function createGeminiClient(): GeminiClient | null;
|
|
47
|
+
//# sourceMappingURL=geminiClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geminiClient.d.ts","sourceRoot":"","sources":["../../src/services/geminiClient.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAA;CACf;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAa;gBAEf,OAAO,EAAE,mBAAmB;IAMxC;;;;OAIG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAQjD;;;OAGG;IACG,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWxD;;;;;OAKG;IACG,YAAY,CAAC,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAwBtE;;;;OAIG;IACG,cAAc,CAAC,UAAU,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAUtD;;;;;OAKG;IACG,eAAe,CAAC,UAAU,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAOtE;;OAEG;IACG,aAAa,CACjB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EACjD,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC;CAyBnB;AAID,wBAAgB,kBAAkB,IAAI,YAAY,GAAG,IAAI,CAYxD"}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { GoogleGenAI } from '@google/genai';
|
|
2
|
+
/**
|
|
3
|
+
* Gemini Client
|
|
4
|
+
* Handles video upload and analysis using Google's Gemini API
|
|
5
|
+
*/
|
|
6
|
+
const GEMINI_MODEL = 'gemini-3-flash-preview';
|
|
7
|
+
export class GeminiClient {
|
|
8
|
+
client;
|
|
9
|
+
constructor(options) {
|
|
10
|
+
this.client = new GoogleGenAI({
|
|
11
|
+
apiKey: options.apiKey,
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Upload a video file to Gemini's storage
|
|
16
|
+
* @param filePath - The local path to the video file
|
|
17
|
+
* @returns File object for use in prompts
|
|
18
|
+
*/
|
|
19
|
+
async uploadVideo(filePath) {
|
|
20
|
+
const fileObject = await this.client.files.upload({
|
|
21
|
+
file: filePath,
|
|
22
|
+
});
|
|
23
|
+
return fileObject;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Wait for a file to finish processing
|
|
27
|
+
* @param name - The file name returned from uploadVideo()
|
|
28
|
+
*/
|
|
29
|
+
async waitForFileProcessing(name) {
|
|
30
|
+
let file = await this.client.files.get({ name });
|
|
31
|
+
while (file.state === 'PROCESSING') {
|
|
32
|
+
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
33
|
+
file = await this.client.files.get({ name });
|
|
34
|
+
}
|
|
35
|
+
if (file.state === 'FAILED') {
|
|
36
|
+
throw new Error('Video processing failed in Gemini API.');
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Analyze a video using the uploaded file object
|
|
41
|
+
* @param fileObject - The file object returned from uploadVideo()
|
|
42
|
+
* @param question - The question to ask about the video
|
|
43
|
+
* @returns The AI analysis response
|
|
44
|
+
*/
|
|
45
|
+
async analyzeVideo(fileObject, question) {
|
|
46
|
+
const response = await this.client.models.generateContent({
|
|
47
|
+
model: GEMINI_MODEL,
|
|
48
|
+
contents: [
|
|
49
|
+
{
|
|
50
|
+
role: 'user',
|
|
51
|
+
parts: [
|
|
52
|
+
{
|
|
53
|
+
fileData: {
|
|
54
|
+
fileUri: fileObject.uri,
|
|
55
|
+
mimeType: fileObject.mimeType,
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
text: question,
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
},
|
|
63
|
+
],
|
|
64
|
+
});
|
|
65
|
+
return response.text || '';
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Summarize a video using the uploaded file object
|
|
69
|
+
* @param fileObject - The file object returned from uploadVideo()
|
|
70
|
+
* @returns The AI summary response
|
|
71
|
+
*/
|
|
72
|
+
async summarizeVideo(fileObject) {
|
|
73
|
+
const prompt = `Please provide a comprehensive summary of this video. Include:
|
|
74
|
+
1. A brief overview of the main topic or events.
|
|
75
|
+
2. Key scenes or moments with approximate timestamps if possible.
|
|
76
|
+
3. Any important dialogue, text, or audio cues.
|
|
77
|
+
4. The overall tone or mood of the video.`;
|
|
78
|
+
return this.analyzeVideo(fileObject, prompt);
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Search for a timestamp in a video using the uploaded file object
|
|
82
|
+
* @param fileObject - The file object returned from uploadVideo()
|
|
83
|
+
* @param query - The event or object to search for
|
|
84
|
+
* @returns The AI search response
|
|
85
|
+
*/
|
|
86
|
+
async searchTimestamp(fileObject, query) {
|
|
87
|
+
const prompt = `Find the exact timestamp(s) where the following occurs in the video: "${query}".
|
|
88
|
+
Please be as precise as possible. If the event occurs multiple times, list all occurrences. If it does not occur, state that clearly.`;
|
|
89
|
+
return this.analyzeVideo(fileObject, prompt);
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Analyze multiple images (e.g., extracted frames)
|
|
93
|
+
*/
|
|
94
|
+
async analyzeImages(images, prompt) {
|
|
95
|
+
const imageParts = images.map((image) => ({
|
|
96
|
+
inlineData: {
|
|
97
|
+
data: image.data,
|
|
98
|
+
mimeType: image.mimeType,
|
|
99
|
+
},
|
|
100
|
+
}));
|
|
101
|
+
const response = await this.client.models.generateContent({
|
|
102
|
+
model: GEMINI_MODEL,
|
|
103
|
+
contents: [
|
|
104
|
+
{
|
|
105
|
+
role: 'user',
|
|
106
|
+
parts: [
|
|
107
|
+
...imageParts,
|
|
108
|
+
{
|
|
109
|
+
text: prompt,
|
|
110
|
+
},
|
|
111
|
+
],
|
|
112
|
+
},
|
|
113
|
+
],
|
|
114
|
+
});
|
|
115
|
+
return response.text || '';
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
let geminiClientInstance = null;
|
|
119
|
+
export function createGeminiClient() {
|
|
120
|
+
if (geminiClientInstance) {
|
|
121
|
+
return geminiClientInstance;
|
|
122
|
+
}
|
|
123
|
+
const apiKey = process.env.GEMINI_API_KEY;
|
|
124
|
+
if (!apiKey) {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
geminiClientInstance = new GeminiClient({ apiKey });
|
|
128
|
+
return geminiClientInstance;
|
|
129
|
+
}
|
|
130
|
+
//# sourceMappingURL=geminiClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geminiClient.js","sourceRoot":"","sources":["../../src/services/geminiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAE3C;;;GAGG;AAEH,MAAM,YAAY,GAAG,wBAAwB,CAAA;AAM7C,MAAM,OAAO,YAAY;IACf,MAAM,CAAa;IAE3B,YAAY,OAA4B;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,WAAW,CAAC;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAA;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,QAAgB;QAChC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAChD,IAAI,EAAE,QAAQ;SACf,CAAC,CAAA;QAEF,OAAO,UAAU,CAAA;IACnB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,qBAAqB,CAAC,IAAY;QACtC,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAChD,OAAO,IAAI,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;YACnC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;YACzD,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9C,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;QAC3D,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,UAAe,EAAE,QAAgB;QAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC;YACxD,KAAK,EAAE,YAAY;YACnB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE;wBACL;4BACE,QAAQ,EAAE;gCACR,OAAO,EAAE,UAAU,CAAC,GAAG;gCACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;6BAC9B;yBACF;wBACD;4BACE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF;aACF;SACF,CAAC,CAAA;QAEF,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAA;IAC5B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAC,UAAe;QAClC,MAAM,MAAM,GAAG;;;;0CAIuB,CAAA;QAEtC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;IAC9C,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe,CAAC,UAAe,EAAE,KAAa;QAClD,MAAM,MAAM,GAAG,yEAAyE,KAAK;sIACqC,CAAA;QAElI,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,MAAiD,EACjD,MAAc;QAEd,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACxC,UAAU,EAAE;gBACV,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB;SACF,CAAC,CAAC,CAAA;QAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC;YACxD,KAAK,EAAE,YAAY;YACnB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE;wBACL,GAAG,UAAU;wBACb;4BACE,IAAI,EAAE,MAAM;yBACb;qBACF;iBACF;aACF;SACF,CAAC,CAAA;QAEF,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAA;IAC5B,CAAC;CACF;AAED,IAAI,oBAAoB,GAAwB,IAAI,CAAA;AAEpD,MAAM,UAAU,kBAAkB;IAChC,IAAI,oBAAoB,EAAE,CAAC;QACzB,OAAO,oBAAoB,CAAA;IAC7B,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAA;IACzC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,IAAI,CAAA;IACb,CAAC;IAED,oBAAoB,GAAG,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;IACnD,OAAO,oBAAoB,CAAA;AAC7B,CAAC"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GLM-4.6V Client
|
|
3
|
+
* Handles video analysis using Z.AI's API
|
|
4
|
+
*/
|
|
5
|
+
export interface GLMClientOptions {
|
|
6
|
+
apiKey: string;
|
|
7
|
+
baseURL?: string;
|
|
8
|
+
}
|
|
9
|
+
export declare class GLMClient {
|
|
10
|
+
private apiKey;
|
|
11
|
+
private baseURL;
|
|
12
|
+
constructor(options: GLMClientOptions);
|
|
13
|
+
/**
|
|
14
|
+
* Analyze a video using base64-encoded data
|
|
15
|
+
* @param base64Data - Base64-encoded video data
|
|
16
|
+
* @param question - The question to ask about the video
|
|
17
|
+
* @param useFlash - Use the free-tier flash model
|
|
18
|
+
* @returns The AI analysis response
|
|
19
|
+
*/
|
|
20
|
+
analyzeVideoBase64(base64Data: string, question: string, useFlash?: boolean): Promise<string>;
|
|
21
|
+
/**
|
|
22
|
+
* Analyze a video from a URL
|
|
23
|
+
* @param url - URL of the video
|
|
24
|
+
* @param question - The question to ask about the video
|
|
25
|
+
* @param useFlash - Use the free-tier flash model
|
|
26
|
+
* @returns The AI analysis response
|
|
27
|
+
*/
|
|
28
|
+
analyzeVideoUrl(url: string, question: string, useFlash?: boolean): Promise<string>;
|
|
29
|
+
/**
|
|
30
|
+
* Analyze multiple images (e.g., extracted frames)
|
|
31
|
+
*/
|
|
32
|
+
analyzeImages(images: Array<{
|
|
33
|
+
data: string;
|
|
34
|
+
mimeType: string;
|
|
35
|
+
}>, prompt: string, useFlash?: boolean): Promise<string>;
|
|
36
|
+
/**
|
|
37
|
+
* Analyze with thinking mode enabled
|
|
38
|
+
*/
|
|
39
|
+
analyzeWithThinking(content: any[], prompt: string, useFlash?: boolean): Promise<{
|
|
40
|
+
reasoning: string;
|
|
41
|
+
answer: string;
|
|
42
|
+
}>;
|
|
43
|
+
private requestChatCompletion;
|
|
44
|
+
private extractTextFromContent;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Create a GLM client instance from environment variables
|
|
48
|
+
*/
|
|
49
|
+
export declare function createGLMClient(): GLMClient | null;
|
|
50
|
+
//# sourceMappingURL=glmClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"glmClient.d.ts","sourceRoot":"","sources":["../../src/services/glmClient.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAwBD,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,OAAO,CAAQ;gBAEX,OAAO,EAAE,gBAAgB;IAKrC;;;;;;OAMG;IACG,kBAAkB,CACtB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,QAAQ,GAAE,OAAc,GACvB,OAAO,CAAC,MAAM,CAAC;IAsBlB;;;;;;OAMG;IACG,eAAe,CACnB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,QAAQ,GAAE,OAAc,GACvB,OAAO,CAAC,MAAM,CAAC;IAoBlB;;OAEG;IACG,aAAa,CACjB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EACjD,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,OAAc,GACvB,OAAO,CAAC,MAAM,CAAC;IAwBlB;;OAEG;IACG,mBAAmB,CACvB,OAAO,EAAE,GAAG,EAAE,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,OAAc,GACvB,OAAO,CAAC;QACT,SAAS,EAAE,MAAM,CAAA;QACjB,MAAM,EAAE,MAAM,CAAA;KACf,CAAC;YAqEY,qBAAqB;IA2BnC,OAAO,CAAC,sBAAsB;CAoB/B;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,SAAS,GAAG,IAAI,CAMlD"}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GLM-4.6V Client
|
|
3
|
+
* Handles video analysis using Z.AI's API
|
|
4
|
+
*/
|
|
5
|
+
const GLM_BASE_URL = 'https://api.z.ai/api/paas/v4/chat/completions';
|
|
6
|
+
const GLM_MODEL = 'glm-4.6v';
|
|
7
|
+
const GLM_MODEL_FLASH = 'glm-4.6v-flash';
|
|
8
|
+
export class GLMClient {
|
|
9
|
+
apiKey;
|
|
10
|
+
baseURL;
|
|
11
|
+
constructor(options) {
|
|
12
|
+
this.apiKey = options.apiKey;
|
|
13
|
+
this.baseURL = options.baseURL || GLM_BASE_URL;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Analyze a video using base64-encoded data
|
|
17
|
+
* @param base64Data - Base64-encoded video data
|
|
18
|
+
* @param question - The question to ask about the video
|
|
19
|
+
* @param useFlash - Use the free-tier flash model
|
|
20
|
+
* @returns The AI analysis response
|
|
21
|
+
*/
|
|
22
|
+
async analyzeVideoBase64(base64Data, question, useFlash = true) {
|
|
23
|
+
const messages = [
|
|
24
|
+
{
|
|
25
|
+
role: 'user',
|
|
26
|
+
content: [
|
|
27
|
+
{
|
|
28
|
+
type: 'video_url',
|
|
29
|
+
video_url: {
|
|
30
|
+
url: `data:video/mp4;base64,${base64Data}`,
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
type: 'text',
|
|
35
|
+
text: question,
|
|
36
|
+
},
|
|
37
|
+
],
|
|
38
|
+
},
|
|
39
|
+
];
|
|
40
|
+
return this.requestChatCompletion(messages, useFlash);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Analyze a video from a URL
|
|
44
|
+
* @param url - URL of the video
|
|
45
|
+
* @param question - The question to ask about the video
|
|
46
|
+
* @param useFlash - Use the free-tier flash model
|
|
47
|
+
* @returns The AI analysis response
|
|
48
|
+
*/
|
|
49
|
+
async analyzeVideoUrl(url, question, useFlash = true) {
|
|
50
|
+
const messages = [
|
|
51
|
+
{
|
|
52
|
+
role: 'user',
|
|
53
|
+
content: [
|
|
54
|
+
{
|
|
55
|
+
type: 'video_url',
|
|
56
|
+
video_url: { url },
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
type: 'text',
|
|
60
|
+
text: question,
|
|
61
|
+
},
|
|
62
|
+
],
|
|
63
|
+
},
|
|
64
|
+
];
|
|
65
|
+
return this.requestChatCompletion(messages, useFlash);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Analyze multiple images (e.g., extracted frames)
|
|
69
|
+
*/
|
|
70
|
+
async analyzeImages(images, prompt, useFlash = true) {
|
|
71
|
+
const imageParts = images.map((image) => ({
|
|
72
|
+
type: 'image_url',
|
|
73
|
+
image_url: {
|
|
74
|
+
url: `data:${image.mimeType};base64,${image.data}`,
|
|
75
|
+
},
|
|
76
|
+
}));
|
|
77
|
+
const messages = [
|
|
78
|
+
{
|
|
79
|
+
role: 'user',
|
|
80
|
+
content: [
|
|
81
|
+
...imageParts,
|
|
82
|
+
{
|
|
83
|
+
type: 'text',
|
|
84
|
+
text: prompt,
|
|
85
|
+
},
|
|
86
|
+
],
|
|
87
|
+
},
|
|
88
|
+
];
|
|
89
|
+
return this.requestChatCompletion(messages, useFlash);
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Analyze with thinking mode enabled
|
|
93
|
+
*/
|
|
94
|
+
async analyzeWithThinking(content, prompt, useFlash = true) {
|
|
95
|
+
const messages = [
|
|
96
|
+
{
|
|
97
|
+
role: 'user',
|
|
98
|
+
content: [
|
|
99
|
+
...content,
|
|
100
|
+
{
|
|
101
|
+
type: 'text',
|
|
102
|
+
text: prompt,
|
|
103
|
+
},
|
|
104
|
+
],
|
|
105
|
+
},
|
|
106
|
+
];
|
|
107
|
+
const basePayload = {
|
|
108
|
+
model: useFlash ? GLM_MODEL_FLASH : GLM_MODEL,
|
|
109
|
+
messages,
|
|
110
|
+
};
|
|
111
|
+
let response = await fetch(this.baseURL, {
|
|
112
|
+
method: 'POST',
|
|
113
|
+
headers: {
|
|
114
|
+
'Content-Type': 'application/json',
|
|
115
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
116
|
+
},
|
|
117
|
+
body: JSON.stringify({
|
|
118
|
+
...basePayload,
|
|
119
|
+
thinking: { type: 'enabled' },
|
|
120
|
+
}),
|
|
121
|
+
});
|
|
122
|
+
let data = (await response.json());
|
|
123
|
+
if (!response.ok) {
|
|
124
|
+
const firstError = data.error?.message || response.statusText;
|
|
125
|
+
const maybeThinkingUnsupported = /thinking|reasoning|unsupported|unknown/i.test(firstError);
|
|
126
|
+
if (!maybeThinkingUnsupported) {
|
|
127
|
+
throw new Error(`GLM API request failed: ${firstError}`);
|
|
128
|
+
}
|
|
129
|
+
response = await fetch(this.baseURL, {
|
|
130
|
+
method: 'POST',
|
|
131
|
+
headers: {
|
|
132
|
+
'Content-Type': 'application/json',
|
|
133
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
134
|
+
},
|
|
135
|
+
body: JSON.stringify(basePayload),
|
|
136
|
+
});
|
|
137
|
+
data = (await response.json());
|
|
138
|
+
if (!response.ok) {
|
|
139
|
+
const fallbackError = data.error?.message || response.statusText;
|
|
140
|
+
throw new Error(`GLM API request failed: ${fallbackError}`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const message = data.choices?.[0]?.message;
|
|
144
|
+
return {
|
|
145
|
+
reasoning: message?.reasoning_content?.trim() ||
|
|
146
|
+
'No reasoning content returned by GLM.',
|
|
147
|
+
answer: this.extractTextFromContent(message?.content),
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
async requestChatCompletion(messages, useFlash) {
|
|
151
|
+
const response = await fetch(this.baseURL, {
|
|
152
|
+
method: 'POST',
|
|
153
|
+
headers: {
|
|
154
|
+
'Content-Type': 'application/json',
|
|
155
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
156
|
+
},
|
|
157
|
+
body: JSON.stringify({
|
|
158
|
+
model: useFlash ? GLM_MODEL_FLASH : GLM_MODEL,
|
|
159
|
+
messages,
|
|
160
|
+
}),
|
|
161
|
+
});
|
|
162
|
+
const data = (await response.json());
|
|
163
|
+
if (!response.ok) {
|
|
164
|
+
const errorMessage = data.error?.message || response.statusText;
|
|
165
|
+
throw new Error(`GLM API request failed: ${errorMessage}`);
|
|
166
|
+
}
|
|
167
|
+
const content = data.choices?.[0]?.message?.content;
|
|
168
|
+
return this.extractTextFromContent(content);
|
|
169
|
+
}
|
|
170
|
+
extractTextFromContent(content) {
|
|
171
|
+
if (typeof content === 'string') {
|
|
172
|
+
return content;
|
|
173
|
+
}
|
|
174
|
+
if (Array.isArray(content)) {
|
|
175
|
+
const text = content
|
|
176
|
+
.filter((part) => part.type === 'text' && typeof part.text === 'string')
|
|
177
|
+
.map((part) => part.text)
|
|
178
|
+
.join('\n');
|
|
179
|
+
if (text.length > 0) {
|
|
180
|
+
return text;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return 'No response content returned by GLM.';
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Create a GLM client instance from environment variables
|
|
188
|
+
*/
|
|
189
|
+
export function createGLMClient() {
|
|
190
|
+
const apiKey = process.env.Z_AI_API_KEY;
|
|
191
|
+
if (!apiKey) {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
return new GLMClient({ apiKey });
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=glmClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"glmClient.js","sourceRoot":"","sources":["../../src/services/glmClient.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,YAAY,GAAG,+CAA+C,CAAA;AACpE,MAAM,SAAS,GAAG,UAAU,CAAA;AAC5B,MAAM,eAAe,GAAG,gBAAgB,CAAA;AA6BxC,MAAM,OAAO,SAAS;IACZ,MAAM,CAAQ;IACd,OAAO,CAAQ;IAEvB,YAAY,OAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAC5B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,YAAY,CAAA;IAChD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CACtB,UAAkB,EAClB,QAAgB,EAChB,WAAoB,IAAI;QAExB,MAAM,QAAQ,GAAiB;YAC7B;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE;4BACT,GAAG,EAAE,yBAAyB,UAAU,EAAE;yBAC3C;qBACF;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF;SACF,CAAA;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACvD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,eAAe,CACnB,GAAW,EACX,QAAgB,EAChB,WAAoB,IAAI;QAExB,MAAM,QAAQ,GAAiB;YAC7B;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,WAAW;wBACjB,SAAS,EAAE,EAAE,GAAG,EAAE;qBACnB;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF;SACF,CAAA;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACvD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,MAAiD,EACjD,MAAc,EACd,WAAoB,IAAI;QAExB,MAAM,UAAU,GAAqB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC1D,IAAI,EAAE,WAAW;YACjB,SAAS,EAAE;gBACT,GAAG,EAAE,QAAQ,KAAK,CAAC,QAAQ,WAAW,KAAK,CAAC,IAAI,EAAE;aACnD;SACF,CAAC,CAAC,CAAA;QAEH,MAAM,QAAQ,GAAiB;YAC7B;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP,GAAG,UAAU;oBACb;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM;qBACb;iBACF;aACF;SACF,CAAA;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACvD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CACvB,OAAc,EACd,MAAc,EACd,WAAoB,IAAI;QAKxB,MAAM,QAAQ,GAAiB;YAC7B;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP,GAAG,OAAO;oBACV;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM;qBACb;iBACF;aACF;SACF,CAAA;QAED,MAAM,WAAW,GAAG;YAClB,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;YAC7C,QAAQ;SACT,CAAA;QAED,IAAI,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE;YACvC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,GAAG,WAAW;gBACd,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;aAC9B,CAAC;SACH,CAAC,CAAA;QAEF,IAAI,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAoB,CAAA;QAErD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAA;YAC7D,MAAM,wBAAwB,GAC5B,yCAAyC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAE5D,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,2BAA2B,UAAU,EAAE,CAAC,CAAA;YAC1D,CAAC;YAED,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE;gBACnC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;aAClC,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAoB,CAAA;YAEjD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAA;gBAChE,MAAM,IAAI,KAAK,CAAC,2BAA2B,aAAa,EAAE,CAAC,CAAA;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAA;QAE1C,OAAO;YACL,SAAS,EACP,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE;gBAClC,uCAAuC;YACzC,MAAM,EAAE,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC;SACtD,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CACjC,QAAsB,EACtB,QAAiB;QAEjB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;gBAC7C,QAAQ;aACT,CAAC;SACH,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAoB,CAAA;QAEvD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAA;YAC/D,MAAM,IAAI,KAAK,CAAC,2BAA2B,YAAY,EAAE,CAAC,CAAA;QAC5D,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAA;QACnD,OAAO,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAA;IAC7C,CAAC;IAEO,sBAAsB,CAC5B,OAAqE;QAErE,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,OAAO,CAAA;QAChB,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,OAAO;iBACjB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;iBACvE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAc,CAAC;iBAClC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEb,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QAED,OAAO,sCAAsC,CAAA;IAC/C,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAA;IACvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;AAClC,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
export interface KimiClientOptions {
|
|
2
|
+
apiKey: string;
|
|
3
|
+
baseURL?: string;
|
|
4
|
+
}
|
|
5
|
+
export declare class KimiClient {
|
|
6
|
+
private client;
|
|
7
|
+
constructor(options: KimiClientOptions);
|
|
8
|
+
/**
|
|
9
|
+
* Upload a video file to Moonshot's storage
|
|
10
|
+
* @param file - The video file (Buffer or ReadableStream)
|
|
11
|
+
* @returns File object with ID for use with ms:// protocol
|
|
12
|
+
*/
|
|
13
|
+
uploadVideo(file: unknown): Promise<{
|
|
14
|
+
id: string;
|
|
15
|
+
object: string;
|
|
16
|
+
purpose: string;
|
|
17
|
+
}>;
|
|
18
|
+
/**
|
|
19
|
+
* Analyze a video using the uploaded file ID
|
|
20
|
+
* @param fileId - The file ID returned from uploadVideo()
|
|
21
|
+
* @param question - The question to ask about the video
|
|
22
|
+
* @returns The AI analysis response
|
|
23
|
+
*/
|
|
24
|
+
analyzeVideo(fileId: string, question: string): Promise<string>;
|
|
25
|
+
/**
|
|
26
|
+
* Analyze video with thinking mode enabled
|
|
27
|
+
*/
|
|
28
|
+
analyzeVideoWithThinking(fileId: string, question: string): Promise<{
|
|
29
|
+
reasoning: string;
|
|
30
|
+
answer: string;
|
|
31
|
+
}>;
|
|
32
|
+
/**
|
|
33
|
+
* Analyze multiple images (e.g., extracted frames)
|
|
34
|
+
*/
|
|
35
|
+
analyzeImages(images: Array<{
|
|
36
|
+
data: string;
|
|
37
|
+
mimeType: string;
|
|
38
|
+
}>, prompt: string): Promise<string>;
|
|
39
|
+
private extractTextFromMessageContent;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Create a Kimi client instance from environment variables
|
|
43
|
+
*/
|
|
44
|
+
export declare function createKimiClient(): KimiClient | null;
|
|
45
|
+
//# sourceMappingURL=kimiClient.d.ts.map
|