shadok-ai 0.14.17 → 0.14.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/telegram.d.ts +96 -11
- package/dist/telegram.js +241 -15
- package/dist/telegram.js.map +1 -1
- package/dist/video.d.ts +103 -0
- package/dist/video.js +181 -0
- package/dist/video.js.map +1 -0
- package/dist/whisper.d.ts +35 -12
- package/dist/whisper.js +58 -8
- package/dist/whisper.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -156,6 +156,14 @@ Enter to skip; you can add it later from the web UI).
|
|
|
156
156
|
`SHADOK_WHISPER_MODEL` picks a bigger one if your recordings need it.
|
|
157
157
|
Set `SHADOK_WHISPER_DIR` to a directory shared between containers and one copy
|
|
158
158
|
serves them all.
|
|
159
|
+
- **Videos on Telegram** — share a video (or a GIF, or a round video note) and
|
|
160
|
+
the agent actually sees it: shadok pulls a handful of stills out of the clip
|
|
161
|
+
for it to look at, and transcribes the sound when there is speech. Both run
|
|
162
|
+
on the same locally-installed toolchain as voice notes, so nothing leaves the
|
|
163
|
+
machine. Clips longer than two minutes keep their stills and the prompt says
|
|
164
|
+
the audio was left unheard — ask and the agent can listen. Telegram caps what
|
|
165
|
+
a bot may download at 20 MB; past that you get a clear notice instead of
|
|
166
|
+
silence.
|
|
159
167
|
- **Run the command an agent asks you to** — when an agent writes `! cmd` for
|
|
160
168
|
you to run (typically because it cannot), a ▶ button under its message runs it
|
|
161
169
|
in the agent's shell after a confirmation showing the exact command. It is
|
package/dist/telegram.d.ts
CHANGED
|
@@ -83,6 +83,42 @@ export declare function nextToolsState(arg: string, current: boolean): boolean;
|
|
|
83
83
|
* Pure — unit tested.
|
|
84
84
|
*/
|
|
85
85
|
export declare function dmGate(owner: number | null, from: number | undefined): "claim" | "allow" | "deny";
|
|
86
|
+
/**
|
|
87
|
+
* The line that tells the agent WHICH message a Telegram reply answers.
|
|
88
|
+
*
|
|
89
|
+
* Replying to something from ten minutes ago is ordinary in a chat, and until
|
|
90
|
+
* now the agent received "fix that" with no trace of the thing being fixed —
|
|
91
|
+
* `reply_to_message` was read only to recover a topic's name.
|
|
92
|
+
*
|
|
93
|
+
* Three decisions worth keeping:
|
|
94
|
+
*
|
|
95
|
+
* The forum trap. In a topic EVERY message carries `reply_to_message`, pointing
|
|
96
|
+
* at the topic's own creation service message. Quoting that would staple a
|
|
97
|
+
* useless line onto every message in every topic, so it is explicitly not a
|
|
98
|
+
* reply.
|
|
99
|
+
*
|
|
100
|
+
* A highlighted fragment wins. Telegram lets the user select part of a message
|
|
101
|
+
* when replying (`quote`, Bot API 7.0). That selection IS them pointing at
|
|
102
|
+
* something; honouring the whole message instead would discard the aim.
|
|
103
|
+
*
|
|
104
|
+
* A bot is "the agent". A bot has a `first_name` too, and quoting the agent's
|
|
105
|
+
* own words under that name reads as a third participant in the conversation.
|
|
106
|
+
*
|
|
107
|
+
* Pure — unit tested.
|
|
108
|
+
*/
|
|
109
|
+
export declare function replyContext(msg: {
|
|
110
|
+
quote?: {
|
|
111
|
+
text?: string;
|
|
112
|
+
};
|
|
113
|
+
reply_to_message?: Record<string, unknown> & {
|
|
114
|
+
from?: {
|
|
115
|
+
is_bot?: boolean;
|
|
116
|
+
first_name?: string;
|
|
117
|
+
last_name?: string;
|
|
118
|
+
username?: string;
|
|
119
|
+
};
|
|
120
|
+
};
|
|
121
|
+
}): string | undefined;
|
|
86
122
|
/**
|
|
87
123
|
* The header of a prompt coming from ELSEWHERE — the web, a cron, the CLI.
|
|
88
124
|
* Without it a Telegram channel only saw the answers: the agent looked like it
|
|
@@ -112,14 +148,30 @@ export interface TgAttachment {
|
|
|
112
148
|
fileUniqueId: string;
|
|
113
149
|
/** `voice` is spoken input, not a file to hand the agent: it is transcribed
|
|
114
150
|
* and the TEXT becomes the prompt. Keeping it a separate kind is what stops
|
|
115
|
-
* an .ogg being dropped on an agent that cannot read audio.
|
|
116
|
-
|
|
151
|
+
* an .ogg being dropped on an agent that cannot read audio. `video` is the
|
|
152
|
+
* same idea one step further: an mp4 is unreadable too, so the bridge turns
|
|
153
|
+
* it into stills (and, when there is speech, a transcript). */
|
|
154
|
+
kind: "image" | "file" | "voice" | "video";
|
|
117
155
|
fileName?: string;
|
|
118
156
|
fileSize?: number;
|
|
157
|
+
mimeType?: string;
|
|
158
|
+
durationSec?: number;
|
|
119
159
|
}
|
|
120
|
-
/**
|
|
121
|
-
*
|
|
122
|
-
*
|
|
160
|
+
/**
|
|
161
|
+
* Extract the attachment of a message — and the list of fields this looks at IS
|
|
162
|
+
* the list of messages the bot can receive at all: `handleMessage` drops
|
|
163
|
+
* anything with neither text nor attachment, so a shape missing here is a
|
|
164
|
+
* message swallowed in silence.
|
|
165
|
+
*
|
|
166
|
+
* That is exactly how a shared VIDEO disappeared: it carries `msg.video`, its
|
|
167
|
+
* caption in `msg.caption`, and no `msg.text` whatsoever. The voice note had
|
|
168
|
+
* been the same bug one shape earlier. So the order below is: spoken input, the
|
|
169
|
+
* moving pictures, then the still ones, then anything else as a file.
|
|
170
|
+
*
|
|
171
|
+
* A `sticker` is deliberately NOT here. It is a reaction, not content, and
|
|
172
|
+
* waking a whole agent turn for an emoji-equivalent costs more than it tells
|
|
173
|
+
* anyone — a decision, not an oversight.
|
|
174
|
+
*/
|
|
123
175
|
export declare function attachmentOf(msg: any): TgAttachment | null;
|
|
124
176
|
/**
|
|
125
177
|
* The file extension for an image pasted into the cockpit, from its
|
|
@@ -139,17 +191,50 @@ export declare function pasteExtension(contentType: string): string;
|
|
|
139
191
|
* type, then a neutral `bin`; the agent reads by content anyway. Pure/testable.
|
|
140
192
|
*/
|
|
141
193
|
export declare function pasteFileName(id: string, name: string, contentType: string): string;
|
|
194
|
+
/** The extension a downloaded attachment should carry, or null when unknown.
|
|
195
|
+
* ffmpeg sniffs the container anyway, so this is for the humans and for the
|
|
196
|
+
* agent reading the path — being truthful matters more than being clever. */
|
|
197
|
+
export declare function mediaExtension(att: TgAttachment): string | null;
|
|
142
198
|
/** Storage name under ~/.shadok-ai/media: keep the original name so Claude
|
|
143
199
|
* has context, prefix with file_unique_id to avoid collisions, and strip
|
|
144
200
|
* anything path-ish or shell-hostile. */
|
|
145
201
|
export declare function mediaFileName(att: TgAttachment): string;
|
|
146
|
-
/**
|
|
147
|
-
|
|
148
|
-
* Bash for the rest), then the user's caption if any. */
|
|
149
|
-
export declare function attachmentPrompt(items: {
|
|
202
|
+
/** One downloaded attachment, as the prompt describes it. */
|
|
203
|
+
export interface PromptItem {
|
|
150
204
|
path: string;
|
|
151
|
-
kind: "image" | "file";
|
|
152
|
-
|
|
205
|
+
kind: "image" | "file" | "video";
|
|
206
|
+
/** Video only: its length, and the stills pulled out of it. */
|
|
207
|
+
durationSec?: number;
|
|
208
|
+
frames?: {
|
|
209
|
+
path: string;
|
|
210
|
+
atSec: number;
|
|
211
|
+
}[];
|
|
212
|
+
/** Video only: the speech heard in the audio track. */
|
|
213
|
+
transcript?: string;
|
|
214
|
+
/** Video only: why there is no transcript, when that is worth saying — an
|
|
215
|
+
* agent that is told nothing about the audio assumes there was none. */
|
|
216
|
+
audioNote?: string;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* The prompt sent to the session for downloaded attachments: one absolute path
|
|
220
|
+
* per line (Claude reads them itself — Read for images/PDF/text, Bash for the
|
|
221
|
+
* rest), then the user's caption if any.
|
|
222
|
+
*
|
|
223
|
+
* A video is the one that needs words around it. The path alone is a dead end
|
|
224
|
+
* (nothing in `Read` opens an mp4), so it comes with its stills — and with the
|
|
225
|
+
* sentence that says to READ them: a model told about a video and handed images
|
|
226
|
+
* next to it still answers "I can't watch videos" unless the link between the
|
|
227
|
+
* two is spelled out. Verified on exactly that failure.
|
|
228
|
+
*/
|
|
229
|
+
export declare function attachmentPrompt(items: PromptItem[], caption?: string): string;
|
|
230
|
+
/** What a bot simply cannot fetch: the Bot API caps `getFile` at 20 MB, and no
|
|
231
|
+
* code on our side lifts it — so the notice names both sizes and the two
|
|
232
|
+
* things that do work. */
|
|
233
|
+
export declare function tooLargeNotice(att: TgAttachment): string;
|
|
234
|
+
/** Was this download refused by the 20 MB ceiling rather than broken? The size
|
|
235
|
+
* we were told is the honest signal; Telegram's own wording is the fallback
|
|
236
|
+
* for a file it never sized. */
|
|
237
|
+
export declare function tooBigFor(att: TgAttachment, err: any): boolean;
|
|
153
238
|
/**
|
|
154
239
|
* Render an agent's Markdown into the small HTML subset Telegram accepts
|
|
155
240
|
* (`parse_mode: "HTML"`): code fences → <pre>, inline `code`, **bold**, *italic*,
|
package/dist/telegram.js
CHANGED
|
@@ -5,6 +5,7 @@ import { loadChannels, upsertChannel, removeChannel, channelForTelegram, homeCha
|
|
|
5
5
|
import { secretNames, setSecret, deleteSecret } from "./secrets.js";
|
|
6
6
|
import { telegramPhotoable } from "./download.js";
|
|
7
7
|
import { transcribe, heardNotice } from "./whisper.js";
|
|
8
|
+
import { clock, lookAtVideo, MAX_FRAMES, ALBUM_FRAMES } from "./video.js";
|
|
8
9
|
import { authStatus, startLogin, submitLoginCode } from "./claude-auth.js";
|
|
9
10
|
import { getProfile, profileNames } from "./profiles.js";
|
|
10
11
|
import { tmuxHasSession } from "./tmux.js";
|
|
@@ -155,6 +156,53 @@ export function dmGate(owner, from) {
|
|
|
155
156
|
return "claim";
|
|
156
157
|
return owner === from ? "allow" : "deny";
|
|
157
158
|
}
|
|
159
|
+
/** How much of a quoted message rides along. It is prepended to EVERY reply, so
|
|
160
|
+
* unbounded it would crowd out the message it exists to contextualise. */
|
|
161
|
+
const QUOTE_MAX = 300;
|
|
162
|
+
/**
|
|
163
|
+
* The line that tells the agent WHICH message a Telegram reply answers.
|
|
164
|
+
*
|
|
165
|
+
* Replying to something from ten minutes ago is ordinary in a chat, and until
|
|
166
|
+
* now the agent received "fix that" with no trace of the thing being fixed —
|
|
167
|
+
* `reply_to_message` was read only to recover a topic's name.
|
|
168
|
+
*
|
|
169
|
+
* Three decisions worth keeping:
|
|
170
|
+
*
|
|
171
|
+
* The forum trap. In a topic EVERY message carries `reply_to_message`, pointing
|
|
172
|
+
* at the topic's own creation service message. Quoting that would staple a
|
|
173
|
+
* useless line onto every message in every topic, so it is explicitly not a
|
|
174
|
+
* reply.
|
|
175
|
+
*
|
|
176
|
+
* A highlighted fragment wins. Telegram lets the user select part of a message
|
|
177
|
+
* when replying (`quote`, Bot API 7.0). That selection IS them pointing at
|
|
178
|
+
* something; honouring the whole message instead would discard the aim.
|
|
179
|
+
*
|
|
180
|
+
* A bot is "the agent". A bot has a `first_name` too, and quoting the agent's
|
|
181
|
+
* own words under that name reads as a third participant in the conversation.
|
|
182
|
+
*
|
|
183
|
+
* Pure — unit tested.
|
|
184
|
+
*/
|
|
185
|
+
export function replyContext(msg) {
|
|
186
|
+
const r = msg.reply_to_message;
|
|
187
|
+
if (!r)
|
|
188
|
+
return undefined;
|
|
189
|
+
// The topic's own creation message: carried by every message in the topic.
|
|
190
|
+
if (r.forum_topic_created)
|
|
191
|
+
return undefined;
|
|
192
|
+
const who = r.from?.is_bot ? "the agent" : (senderName(r.from) ?? "someone");
|
|
193
|
+
const picked = (msg.quote?.text ?? r.text ?? r.caption ?? "").trim();
|
|
194
|
+
if (picked) {
|
|
195
|
+
// One line: the quote sits above the message, and a multi-line one would
|
|
196
|
+
// read as part of what the human wrote.
|
|
197
|
+
const flat = picked.replace(/\s+/g, " ");
|
|
198
|
+
const cut = flat.length > QUOTE_MAX ? flat.slice(0, QUOTE_MAX) + "…" : flat;
|
|
199
|
+
return `⟦replying to ${who}⟧ "${cut}"`;
|
|
200
|
+
}
|
|
201
|
+
// Nothing quotable: name the medium rather than print an empty quote, which
|
|
202
|
+
// reads as a rendering bug.
|
|
203
|
+
const medium = r.photo ? "a photo" : r.document ? "a file" : r.voice ? "a voice message" : r.video ? "a video" : r.sticker ? "a sticker" : "a message";
|
|
204
|
+
return `⟦replying to ${who}⟧ [${medium}]`;
|
|
205
|
+
}
|
|
158
206
|
/** The known origins: the icon that stands for them, and the word they print.
|
|
159
207
|
* An unknown origin keeps its own name: better "someone spoke" than a message
|
|
160
208
|
* that looks like it came from the agent. */
|
|
@@ -206,9 +254,21 @@ export function migratedGroupId(msg, currentBound) {
|
|
|
206
254
|
return null;
|
|
207
255
|
return msg?.chat?.id === currentBound ? newId : null;
|
|
208
256
|
}
|
|
209
|
-
/**
|
|
210
|
-
*
|
|
211
|
-
*
|
|
257
|
+
/**
|
|
258
|
+
* Extract the attachment of a message — and the list of fields this looks at IS
|
|
259
|
+
* the list of messages the bot can receive at all: `handleMessage` drops
|
|
260
|
+
* anything with neither text nor attachment, so a shape missing here is a
|
|
261
|
+
* message swallowed in silence.
|
|
262
|
+
*
|
|
263
|
+
* That is exactly how a shared VIDEO disappeared: it carries `msg.video`, its
|
|
264
|
+
* caption in `msg.caption`, and no `msg.text` whatsoever. The voice note had
|
|
265
|
+
* been the same bug one shape earlier. So the order below is: spoken input, the
|
|
266
|
+
* moving pictures, then the still ones, then anything else as a file.
|
|
267
|
+
*
|
|
268
|
+
* A `sticker` is deliberately NOT here. It is a reaction, not content, and
|
|
269
|
+
* waking a whole agent turn for an emoji-equivalent costs more than it tells
|
|
270
|
+
* anyone — a decision, not an oversight.
|
|
271
|
+
*/
|
|
212
272
|
export function attachmentOf(msg) {
|
|
213
273
|
// Voice FIRST: a voice note also carries no photo and no document, and used
|
|
214
274
|
// to fall through to `null` — so the message was dropped in silence, which is
|
|
@@ -220,20 +280,58 @@ export function attachmentOf(msg) {
|
|
|
220
280
|
fileUniqueId: v.file_unique_id,
|
|
221
281
|
kind: "voice",
|
|
222
282
|
...(v.file_size != null ? { fileSize: v.file_size } : {}),
|
|
283
|
+
...(v.duration != null ? { durationSec: v.duration } : {}),
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
// A video, a GIF (`animation` — Telegram stores it as a soundless mp4) or a
|
|
287
|
+
// round "telescope" clip (`video_note`). `animation` comes BEFORE `document`
|
|
288
|
+
// on purpose: Telegram sends both fields for a GIF, and as a document it
|
|
289
|
+
// would lose its duration and its stills.
|
|
290
|
+
const mov = msg.video ?? msg.animation ?? msg.video_note;
|
|
291
|
+
if (mov?.file_id) {
|
|
292
|
+
return {
|
|
293
|
+
fileId: mov.file_id,
|
|
294
|
+
fileUniqueId: mov.file_unique_id,
|
|
295
|
+
kind: "video",
|
|
296
|
+
...(mov.file_name ? { fileName: mov.file_name } : {}),
|
|
297
|
+
...(mov.file_size != null ? { fileSize: mov.file_size } : {}),
|
|
298
|
+
...(mov.mime_type ? { mimeType: mov.mime_type } : {}),
|
|
299
|
+
...(mov.duration != null ? { durationSec: mov.duration } : {}),
|
|
223
300
|
};
|
|
224
301
|
}
|
|
225
302
|
if (Array.isArray(msg.photo) && msg.photo.length) {
|
|
226
303
|
const p = msg.photo[msg.photo.length - 1];
|
|
227
304
|
return { fileId: p.file_id, fileUniqueId: p.file_unique_id, kind: "image", fileSize: p.file_size };
|
|
228
305
|
}
|
|
306
|
+
// An audio FILE (a music track, a forwarded memo) — handed over as a file
|
|
307
|
+
// rather than transcribed: a 40-minute track would hold a turn hostage for
|
|
308
|
+
// the best part of an hour, and nobody shares an album for its lyrics. The
|
|
309
|
+
// point here is that it stops vanishing.
|
|
310
|
+
const a = msg.audio;
|
|
311
|
+
if (a?.file_id) {
|
|
312
|
+
return {
|
|
313
|
+
fileId: a.file_id,
|
|
314
|
+
fileUniqueId: a.file_unique_id,
|
|
315
|
+
kind: "file",
|
|
316
|
+
...(a.file_name ? { fileName: a.file_name } : {}),
|
|
317
|
+
...(a.file_size != null ? { fileSize: a.file_size } : {}),
|
|
318
|
+
...(a.mime_type ? { mimeType: a.mime_type } : {}),
|
|
319
|
+
...(a.duration != null ? { durationSec: a.duration } : {}),
|
|
320
|
+
};
|
|
321
|
+
}
|
|
229
322
|
const d = msg.document;
|
|
230
323
|
if (d?.file_id) {
|
|
231
324
|
return {
|
|
232
325
|
fileId: d.file_id,
|
|
233
326
|
fileUniqueId: d.file_unique_id,
|
|
234
|
-
kind: typeof d.mime_type === "string" && d.mime_type.startsWith("
|
|
327
|
+
kind: typeof d.mime_type === "string" && d.mime_type.startsWith("video/")
|
|
328
|
+
? "video"
|
|
329
|
+
: typeof d.mime_type === "string" && d.mime_type.startsWith("image/")
|
|
330
|
+
? "image"
|
|
331
|
+
: "file",
|
|
235
332
|
...(d.file_name ? { fileName: d.file_name } : {}),
|
|
236
333
|
...(d.file_size != null ? { fileSize: d.file_size } : {}),
|
|
334
|
+
...(d.mime_type ? { mimeType: d.mime_type } : {}),
|
|
237
335
|
};
|
|
238
336
|
}
|
|
239
337
|
return null;
|
|
@@ -283,6 +381,40 @@ export function pasteFileName(id, name, contentType) {
|
|
|
283
381
|
const ext = pasteExtension(contentType);
|
|
284
382
|
return base ? `paste-${id}-${base}.${ext}` : `paste-${id}.${ext}`;
|
|
285
383
|
}
|
|
384
|
+
/**
|
|
385
|
+
* A WHITELIST from Telegram's mime type to an extension, for the media that
|
|
386
|
+
* arrives without a file name — a camera video has none.
|
|
387
|
+
*
|
|
388
|
+
* Same rule as `pasteExtension`: never split on "/", since the result lands in
|
|
389
|
+
* a filename. Unknown → null, and the caller decides what is honest.
|
|
390
|
+
*/
|
|
391
|
+
const MEDIA_EXT = {
|
|
392
|
+
"video/mp4": "mp4",
|
|
393
|
+
"video/quicktime": "mov",
|
|
394
|
+
"video/webm": "webm",
|
|
395
|
+
"video/x-matroska": "mkv",
|
|
396
|
+
"video/mpeg": "mpeg",
|
|
397
|
+
"audio/mpeg": "mp3",
|
|
398
|
+
"audio/mp4": "m4a",
|
|
399
|
+
"audio/ogg": "ogg",
|
|
400
|
+
"audio/x-wav": "wav",
|
|
401
|
+
"audio/wav": "wav",
|
|
402
|
+
};
|
|
403
|
+
/** The extension a downloaded attachment should carry, or null when unknown.
|
|
404
|
+
* ffmpeg sniffs the container anyway, so this is for the humans and for the
|
|
405
|
+
* agent reading the path — being truthful matters more than being clever. */
|
|
406
|
+
export function mediaExtension(att) {
|
|
407
|
+
const fromMime = att.mimeType ? MEDIA_EXT[att.mimeType.split(";")[0].trim().toLowerCase()] : undefined;
|
|
408
|
+
if (fromMime)
|
|
409
|
+
return fromMime;
|
|
410
|
+
if (att.kind === "image")
|
|
411
|
+
return "jpg";
|
|
412
|
+
// A video Telegram transcoded itself is always mp4; saying nothing would
|
|
413
|
+
// leave the file extension-less, which reads as "unknown binary".
|
|
414
|
+
if (att.kind === "video")
|
|
415
|
+
return "mp4";
|
|
416
|
+
return null;
|
|
417
|
+
}
|
|
286
418
|
/** Storage name under ~/.shadok-ai/media: keep the original name so Claude
|
|
287
419
|
* has context, prefix with file_unique_id to avoid collisions, and strip
|
|
288
420
|
* anything path-ish or shell-hostile. */
|
|
@@ -290,15 +422,56 @@ export function mediaFileName(att) {
|
|
|
290
422
|
const base = att.fileName ? path.basename(att.fileName).replace(/[^\w.\- ]+/g, "_") : "";
|
|
291
423
|
if (base)
|
|
292
424
|
return `${att.fileUniqueId}-${base}`;
|
|
293
|
-
|
|
425
|
+
const ext = mediaExtension(att);
|
|
426
|
+
return ext ? `${att.fileUniqueId}.${ext}` : att.fileUniqueId;
|
|
294
427
|
}
|
|
295
|
-
/**
|
|
296
|
-
*
|
|
297
|
-
*
|
|
428
|
+
/**
|
|
429
|
+
* The prompt sent to the session for downloaded attachments: one absolute path
|
|
430
|
+
* per line (Claude reads them itself — Read for images/PDF/text, Bash for the
|
|
431
|
+
* rest), then the user's caption if any.
|
|
432
|
+
*
|
|
433
|
+
* A video is the one that needs words around it. The path alone is a dead end
|
|
434
|
+
* (nothing in `Read` opens an mp4), so it comes with its stills — and with the
|
|
435
|
+
* sentence that says to READ them: a model told about a video and handed images
|
|
436
|
+
* next to it still answers "I can't watch videos" unless the link between the
|
|
437
|
+
* two is spelled out. Verified on exactly that failure.
|
|
438
|
+
*/
|
|
298
439
|
export function attachmentPrompt(items, caption) {
|
|
299
|
-
const lines =
|
|
440
|
+
const lines = [];
|
|
441
|
+
for (const i of items) {
|
|
442
|
+
if (i.kind === "video") {
|
|
443
|
+
const len = i.durationSec ? ` (${clock(i.durationSec)})` : "";
|
|
444
|
+
lines.push(`[Attached video: ${i.path}${len}]`);
|
|
445
|
+
if (i.frames?.length) {
|
|
446
|
+
lines.push("Stills from it — read these images to see the video:");
|
|
447
|
+
i.frames.forEach((f, n) => lines.push(`[Attached image: ${f.path}] frame ${n + 1}/${i.frames.length} at ${clock(f.atSec)}`));
|
|
448
|
+
}
|
|
449
|
+
if (i.transcript?.trim())
|
|
450
|
+
lines.push(`⟦heard in the video⟧ "${i.transcript.trim()}"`);
|
|
451
|
+
else if (i.audioNote)
|
|
452
|
+
lines.push(`⟦video audio⟧ ${i.audioNote}`);
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
455
|
+
lines.push(i.kind === "image" ? `[Attached image: ${i.path}]` : `[Attached file: ${i.path}]`);
|
|
456
|
+
}
|
|
300
457
|
return caption?.trim() ? lines.join("\n") + "\n" + caption : lines.join("\n");
|
|
301
458
|
}
|
|
459
|
+
/** What a bot simply cannot fetch: the Bot API caps `getFile` at 20 MB, and no
|
|
460
|
+
* code on our side lifts it — so the notice names both sizes and the two
|
|
461
|
+
* things that do work. */
|
|
462
|
+
export function tooLargeNotice(att) {
|
|
463
|
+
const mb = att.fileSize ? `${(att.fileSize / 1_000_000).toFixed(1)} MB` : "this file";
|
|
464
|
+
const what = att.kind === "video" ? "video" : att.kind === "image" ? "image" : "file";
|
|
465
|
+
return `⚠️ Telegram only lets a bot download up to 20 MB, and this ${what} is ${mb}. Send a shorter clip, or compress it — a link to the file works too.`;
|
|
466
|
+
}
|
|
467
|
+
/** Was this download refused by the 20 MB ceiling rather than broken? The size
|
|
468
|
+
* we were told is the honest signal; Telegram's own wording is the fallback
|
|
469
|
+
* for a file it never sized. */
|
|
470
|
+
export function tooBigFor(att, err) {
|
|
471
|
+
if (att.fileSize && att.fileSize > TG_FILE_LIMIT)
|
|
472
|
+
return true;
|
|
473
|
+
return /too (big|large)/i.test(String(err?.message ?? err ?? ""));
|
|
474
|
+
}
|
|
302
475
|
const htmlEscape = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
303
476
|
/**
|
|
304
477
|
* Render an agent's Markdown into the small HTML subset Telegram accepts
|
|
@@ -894,7 +1067,7 @@ export function startTelegram(port, authCookie) {
|
|
|
894
1067
|
const text = b.preempting;
|
|
895
1068
|
b.preempting = undefined;
|
|
896
1069
|
clearTimeout(b.preemptTimer);
|
|
897
|
-
promptTo(b, text,
|
|
1070
|
+
promptTo(b, text, b.lastSentFrom, true);
|
|
898
1071
|
break;
|
|
899
1072
|
}
|
|
900
1073
|
b.typing.stop();
|
|
@@ -973,6 +1146,9 @@ export function startTelegram(port, authCookie) {
|
|
|
973
1146
|
// name the author instead of showing an anonymous "pilot (elsewhere)".
|
|
974
1147
|
const promptTo = (b, text, from, resend = false, voice = false) => {
|
|
975
1148
|
b.lastSent = text;
|
|
1149
|
+
// A resend must keep the original author; only a NEW message sets one.
|
|
1150
|
+
if (!resend)
|
|
1151
|
+
b.lastSentFrom = from;
|
|
976
1152
|
// Un message NEUF rouvre le droit d'interrompre ; un renvoi, non — sinon
|
|
977
1153
|
// deux clients qui se disputent la session se relancent indéfiniment.
|
|
978
1154
|
if (!resend)
|
|
@@ -1012,10 +1188,30 @@ export function startTelegram(port, authCookie) {
|
|
|
1012
1188
|
// read audio, and a defensive skip is cheaper than finding out.
|
|
1013
1189
|
if (i.att.kind === "voice")
|
|
1014
1190
|
continue;
|
|
1015
|
-
|
|
1191
|
+
const p = await downloadAttachment(i.att);
|
|
1192
|
+
const item = { path: p, kind: i.att.kind };
|
|
1193
|
+
if (i.att.kind === "video") {
|
|
1194
|
+
// Two stills per clip and no transcription: ten videos in one album
|
|
1195
|
+
// would otherwise mean sixty images and several minutes of whisper
|
|
1196
|
+
// before the agent is even woken.
|
|
1197
|
+
const look = await lookAtVideo(p, {
|
|
1198
|
+
durationSec: i.att.durationSec,
|
|
1199
|
+
maxFrames: ALBUM_FRAMES,
|
|
1200
|
+
transcribe: false,
|
|
1201
|
+
notify: (line) => void reply(b.chatId, b.threadId, line),
|
|
1202
|
+
});
|
|
1203
|
+
Object.assign(item, {
|
|
1204
|
+
...(look.durationSec ? { durationSec: look.durationSec } : {}),
|
|
1205
|
+
...(look.frames ? { frames: look.frames } : {}),
|
|
1206
|
+
...(look.audioNote ? { audioNote: look.audioNote } : {}),
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
1209
|
+
ok.push(item);
|
|
1016
1210
|
}
|
|
1017
1211
|
catch (e) {
|
|
1018
|
-
failed.push(
|
|
1212
|
+
failed.push(tooBigFor(i.att, e)
|
|
1213
|
+
? `${i.att.fileName ?? i.att.fileUniqueId} (over the 20 MB Telegram ceiling)`
|
|
1214
|
+
: `${i.att.fileName ?? i.att.fileUniqueId} (${e?.message ?? e})`);
|
|
1019
1215
|
}
|
|
1020
1216
|
}
|
|
1021
1217
|
if (failed.length)
|
|
@@ -1405,11 +1601,38 @@ export function startTelegram(port, authCookie) {
|
|
|
1405
1601
|
promptTo(b, caption ? `${out.text}\n\n${caption}` : out.text, senderName(msg.from), false, true);
|
|
1406
1602
|
return;
|
|
1407
1603
|
}
|
|
1408
|
-
|
|
1604
|
+
const attQuote = replyContext(msg);
|
|
1605
|
+
// A video is not readable as a file: it is looked AT (stills) and
|
|
1606
|
+
// listened to (its audio track), both over the ffmpeg the voice
|
|
1607
|
+
// pipeline already provisions. Without this the agent received a path
|
|
1608
|
+
// to an mp4 and answered that it cannot watch videos.
|
|
1609
|
+
const item = { path: p, kind: att.kind };
|
|
1610
|
+
if (att.kind === "video") {
|
|
1611
|
+
const look = await lookAtVideo(p, {
|
|
1612
|
+
durationSec: att.durationSec,
|
|
1613
|
+
maxFrames: MAX_FRAMES,
|
|
1614
|
+
notify: (line) => void reply(chat.id, threadId, line),
|
|
1615
|
+
});
|
|
1616
|
+
if (look.trouble)
|
|
1617
|
+
await reply(chat.id, threadId, look.trouble);
|
|
1618
|
+
Object.assign(item, {
|
|
1619
|
+
...(look.durationSec ? { durationSec: look.durationSec } : {}),
|
|
1620
|
+
...(look.frames ? { frames: look.frames } : {}),
|
|
1621
|
+
...(look.transcript ? { transcript: look.transcript } : {}),
|
|
1622
|
+
...(look.audioNote ? { audioNote: look.audioNote } : {}),
|
|
1623
|
+
});
|
|
1624
|
+
}
|
|
1625
|
+
const attText = attachmentPrompt([item], caption);
|
|
1626
|
+
promptTo(b, attQuote ? attQuote + "\n" + attText : attText, senderName(msg.from));
|
|
1409
1627
|
}
|
|
1410
1628
|
catch (e) {
|
|
1411
1629
|
b.typing.stop();
|
|
1412
|
-
|
|
1630
|
+
// The 20 MB ceiling is the common case for a phone video, and it is not
|
|
1631
|
+
// an error on anyone's side — so it gets its own actionable wording
|
|
1632
|
+
// instead of a raw message the user can do nothing with.
|
|
1633
|
+
await reply(chat.id, threadId, tooBigFor(att, e)
|
|
1634
|
+
? tooLargeNotice(att)
|
|
1635
|
+
: `⚠️ could not download ${att.fileName ?? "the attachment"} (${e?.message ?? e})`);
|
|
1413
1636
|
}
|
|
1414
1637
|
return;
|
|
1415
1638
|
}
|
|
@@ -1426,7 +1649,10 @@ export function startTelegram(port, authCookie) {
|
|
|
1426
1649
|
b.pendingActions.push(wsMsg);
|
|
1427
1650
|
return;
|
|
1428
1651
|
}
|
|
1429
|
-
|
|
1652
|
+
// A reply carries the message it answers: without it the agent reads
|
|
1653
|
+
// "fix that" with no trace of what "that" is.
|
|
1654
|
+
const quoted = replyContext(msg);
|
|
1655
|
+
promptTo(b, quoted ? quoted + "\n" + msg.text : msg.text, senderName(msg.from));
|
|
1430
1656
|
};
|
|
1431
1657
|
const handleCallback = async (cq) => {
|
|
1432
1658
|
const action = parseCallback(cq.data ?? "");
|