shadok-ai 0.14.17 → 0.14.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -156,6 +156,14 @@ Enter to skip; you can add it later from the web UI).
156
156
  `SHADOK_WHISPER_MODEL` picks a bigger one if your recordings need it.
157
157
  Set `SHADOK_WHISPER_DIR` to a directory shared between containers and one copy
158
158
  serves them all.
159
+ - **Videos on Telegram** — share a video (or a GIF, or a round video note) and
160
+ the agent actually sees it: shadok pulls a handful of stills out of the clip
161
+ for it to look at, and transcribes the sound when there is speech. Both run
162
+ on the same locally-installed toolchain as voice notes, so nothing leaves the
163
+ machine. Clips longer than two minutes keep their stills and the prompt says
164
+ the audio was left unheard — ask and the agent can listen. Telegram caps what
165
+ a bot may download at 20 MB; past that you get a clear notice instead of
166
+ silence.
159
167
  - **Run the command an agent asks you to** — when an agent writes `! cmd` for
160
168
  you to run (typically because it cannot), a ▶ button under its message runs it
161
169
  in the agent's shell after a confirmation showing the exact command. It is
@@ -83,6 +83,42 @@ export declare function nextToolsState(arg: string, current: boolean): boolean;
83
83
  * Pure — unit tested.
84
84
  */
85
85
  export declare function dmGate(owner: number | null, from: number | undefined): "claim" | "allow" | "deny";
86
+ /**
87
+ * The line that tells the agent WHICH message a Telegram reply answers.
88
+ *
89
+ * Replying to something from ten minutes ago is ordinary in a chat, and until
90
+ * now the agent received "fix that" with no trace of the thing being fixed —
91
+ * `reply_to_message` was read only to recover a topic's name.
92
+ *
93
+ * Three decisions worth keeping:
94
+ *
95
+ * The forum trap. In a topic EVERY message carries `reply_to_message`, pointing
96
+ * at the topic's own creation service message. Quoting that would staple a
97
+ * useless line onto every message in every topic, so it is explicitly not a
98
+ * reply.
99
+ *
100
+ * A highlighted fragment wins. Telegram lets the user select part of a message
101
+ * when replying (`quote`, Bot API 7.0). That selection IS them pointing at
102
+ * something; honouring the whole message instead would discard the aim.
103
+ *
104
+ * A bot is "the agent". A bot has a `first_name` too, and quoting the agent's
105
+ * own words under that name reads as a third participant in the conversation.
106
+ *
107
+ * Pure — unit tested.
108
+ */
109
+ export declare function replyContext(msg: {
110
+ quote?: {
111
+ text?: string;
112
+ };
113
+ reply_to_message?: Record<string, unknown> & {
114
+ from?: {
115
+ is_bot?: boolean;
116
+ first_name?: string;
117
+ last_name?: string;
118
+ username?: string;
119
+ };
120
+ };
121
+ }): string | undefined;
86
122
  /**
87
123
  * The header of a prompt coming from ELSEWHERE — the web, a cron, the CLI.
88
124
  * Without it a Telegram channel only saw the answers: the agent looked like it
@@ -112,14 +148,30 @@ export interface TgAttachment {
112
148
  fileUniqueId: string;
113
149
  /** `voice` is spoken input, not a file to hand the agent: it is transcribed
114
150
  * and the TEXT becomes the prompt. Keeping it a separate kind is what stops
115
- * an .ogg being dropped on an agent that cannot read audio. */
116
- kind: "image" | "file" | "voice";
151
+ * an .ogg being dropped on an agent that cannot read audio. `video` is the
152
+ * same idea one step further: an mp4 is unreadable too, so the bridge turns
153
+ * it into stills (and, when there is speech, a transcript). */
154
+ kind: "image" | "file" | "voice" | "video";
117
155
  fileName?: string;
118
156
  fileSize?: number;
157
+ mimeType?: string;
158
+ durationSec?: number;
119
159
  }
120
- /** Extract the attachment of a message: a photo (largest size — Telegram
121
- * sorts sizes small → large) or any document (PDF, zip, image sent as
122
- * file…). Text-only messages → null. */
160
+ /**
161
+ * Extract the attachment of a message — and the list of fields this looks at IS
162
+ * the list of messages the bot can receive at all: `handleMessage` drops
163
+ * anything with neither text nor attachment, so a shape missing here is a
164
+ * message swallowed in silence.
165
+ *
166
+ * That is exactly how a shared VIDEO disappeared: it carries `msg.video`, its
167
+ * caption in `msg.caption`, and no `msg.text` whatsoever. The voice note had
168
+ * been the same bug one shape earlier. So the order below is: spoken input, the
169
+ * moving pictures, then the still ones, then anything else as a file.
170
+ *
171
+ * A `sticker` is deliberately NOT here. It is a reaction, not content, and
172
+ * waking a whole agent turn for an emoji-equivalent costs more than it tells
173
+ * anyone — a decision, not an oversight.
174
+ */
123
175
  export declare function attachmentOf(msg: any): TgAttachment | null;
124
176
  /**
125
177
  * The file extension for an image pasted into the cockpit, from its
@@ -139,17 +191,50 @@ export declare function pasteExtension(contentType: string): string;
139
191
  * type, then a neutral `bin`; the agent reads by content anyway. Pure/testable.
140
192
  */
141
193
  export declare function pasteFileName(id: string, name: string, contentType: string): string;
194
+ /** The extension a downloaded attachment should carry, or null when unknown.
195
+ * ffmpeg sniffs the container anyway, so this is for the humans and for the
196
+ * agent reading the path — being truthful matters more than being clever. */
197
+ export declare function mediaExtension(att: TgAttachment): string | null;
142
198
  /** Storage name under ~/.shadok-ai/media: keep the original name so Claude
143
199
  * has context, prefix with file_unique_id to avoid collisions, and strip
144
200
  * anything path-ish or shell-hostile. */
145
201
  export declare function mediaFileName(att: TgAttachment): string;
146
- /** The prompt sent to the session for downloaded attachments: one absolute
147
- * path per line (Claude reads them itself — Read for images/PDF/text,
148
- * Bash for the rest), then the user's caption if any. */
149
- export declare function attachmentPrompt(items: {
202
+ /** One downloaded attachment, as the prompt describes it. */
203
+ export interface PromptItem {
150
204
  path: string;
151
- kind: "image" | "file";
152
- }[], caption?: string): string;
205
+ kind: "image" | "file" | "video";
206
+ /** Video only: its length, and the stills pulled out of it. */
207
+ durationSec?: number;
208
+ frames?: {
209
+ path: string;
210
+ atSec: number;
211
+ }[];
212
+ /** Video only: the speech heard in the audio track. */
213
+ transcript?: string;
214
+ /** Video only: why there is no transcript, when that is worth saying — an
215
+ * agent that is told nothing about the audio assumes there was none. */
216
+ audioNote?: string;
217
+ }
218
+ /**
219
+ * The prompt sent to the session for downloaded attachments: one absolute path
220
+ * per line (Claude reads them itself — Read for images/PDF/text, Bash for the
221
+ * rest), then the user's caption if any.
222
+ *
223
+ * A video is the one that needs words around it. The path alone is a dead end
224
+ * (nothing in `Read` opens an mp4), so it comes with its stills — and with the
225
+ * sentence that says to READ them: a model told about a video and handed images
226
+ * next to it still answers "I can't watch videos" unless the link between the
227
+ * two is spelled out. Verified on exactly that failure.
228
+ */
229
+ export declare function attachmentPrompt(items: PromptItem[], caption?: string): string;
230
+ /** What a bot simply cannot fetch: the Bot API caps `getFile` at 20 MB, and no
231
+ * code on our side lifts it — so the notice names both sizes and the two
232
+ * things that do work. */
233
+ export declare function tooLargeNotice(att: TgAttachment): string;
234
+ /** Was this download refused by the 20 MB ceiling rather than broken? The size
235
+ * we were told is the honest signal; Telegram's own wording is the fallback
236
+ * for a file it never sized. */
237
+ export declare function tooBigFor(att: TgAttachment, err: any): boolean;
153
238
  /**
154
239
  * Render an agent's Markdown into the small HTML subset Telegram accepts
155
240
  * (`parse_mode: "HTML"`): code fences → <pre>, inline `code`, **bold**, *italic*,
package/dist/telegram.js CHANGED
@@ -5,6 +5,7 @@ import { loadChannels, upsertChannel, removeChannel, channelForTelegram, homeCha
5
5
  import { secretNames, setSecret, deleteSecret } from "./secrets.js";
6
6
  import { telegramPhotoable } from "./download.js";
7
7
  import { transcribe, heardNotice } from "./whisper.js";
8
+ import { clock, lookAtVideo, MAX_FRAMES, ALBUM_FRAMES } from "./video.js";
8
9
  import { authStatus, startLogin, submitLoginCode } from "./claude-auth.js";
9
10
  import { getProfile, profileNames } from "./profiles.js";
10
11
  import { tmuxHasSession } from "./tmux.js";
@@ -155,6 +156,53 @@ export function dmGate(owner, from) {
155
156
  return "claim";
156
157
  return owner === from ? "allow" : "deny";
157
158
  }
159
+ /** How much of a quoted message rides along. It is prepended to EVERY reply, so
160
+ * unbounded it would crowd out the message it exists to contextualise. */
161
+ const QUOTE_MAX = 300;
162
+ /**
163
+ * The line that tells the agent WHICH message a Telegram reply answers.
164
+ *
165
+ * Replying to something from ten minutes ago is ordinary in a chat, and until
166
+ * now the agent received "fix that" with no trace of the thing being fixed —
167
+ * `reply_to_message` was read only to recover a topic's name.
168
+ *
169
+ * Three decisions worth keeping:
170
+ *
171
+ * The forum trap. In a topic EVERY message carries `reply_to_message`, pointing
172
+ * at the topic's own creation service message. Quoting that would staple a
173
+ * useless line onto every message in every topic, so it is explicitly not a
174
+ * reply.
175
+ *
176
+ * A highlighted fragment wins. Telegram lets the user select part of a message
177
+ * when replying (`quote`, Bot API 7.0). That selection IS them pointing at
178
+ * something; honouring the whole message instead would discard the aim.
179
+ *
180
+ * A bot is "the agent". A bot has a `first_name` too, and quoting the agent's
181
+ * own words under that name reads as a third participant in the conversation.
182
+ *
183
+ * Pure — unit tested.
184
+ */
185
+ export function replyContext(msg) {
186
+ const r = msg.reply_to_message;
187
+ if (!r)
188
+ return undefined;
189
+ // The topic's own creation message: carried by every message in the topic.
190
+ if (r.forum_topic_created)
191
+ return undefined;
192
+ const who = r.from?.is_bot ? "the agent" : (senderName(r.from) ?? "someone");
193
+ const picked = (msg.quote?.text ?? r.text ?? r.caption ?? "").trim();
194
+ if (picked) {
195
+ // One line: the quote sits above the message, and a multi-line one would
196
+ // read as part of what the human wrote.
197
+ const flat = picked.replace(/\s+/g, " ");
198
+ const cut = flat.length > QUOTE_MAX ? flat.slice(0, QUOTE_MAX) + "…" : flat;
199
+ return `⟦replying to ${who}⟧ "${cut}"`;
200
+ }
201
+ // Nothing quotable: name the medium rather than print an empty quote, which
202
+ // reads as a rendering bug.
203
+ const medium = r.photo ? "a photo" : r.document ? "a file" : r.voice ? "a voice message" : r.video ? "a video" : r.sticker ? "a sticker" : "a message";
204
+ return `⟦replying to ${who}⟧ [${medium}]`;
205
+ }
158
206
  /** The known origins: the icon that stands for them, and the word they print.
159
207
  * An unknown origin keeps its own name: better "someone spoke" than a message
160
208
  * that looks like it came from the agent. */
@@ -206,9 +254,21 @@ export function migratedGroupId(msg, currentBound) {
206
254
  return null;
207
255
  return msg?.chat?.id === currentBound ? newId : null;
208
256
  }
209
- /** Extract the attachment of a message: a photo (largest size — Telegram
210
- * sorts sizes small → large) or any document (PDF, zip, image sent as
211
- * file…). Text-only messages → null. */
257
+ /**
258
+ * Extract the attachment of a message — and the list of fields this looks at IS
259
+ * the list of messages the bot can receive at all: `handleMessage` drops
260
+ * anything with neither text nor attachment, so a shape missing here is a
261
+ * message swallowed in silence.
262
+ *
263
+ * That is exactly how a shared VIDEO disappeared: it carries `msg.video`, its
264
+ * caption in `msg.caption`, and no `msg.text` whatsoever. The voice note had
265
+ * been the same bug one shape earlier. So the order below is: spoken input, the
266
+ * moving pictures, then the still ones, then anything else as a file.
267
+ *
268
+ * A `sticker` is deliberately NOT here. It is a reaction, not content, and
269
+ * waking a whole agent turn for an emoji-equivalent costs more than it tells
270
+ * anyone — a decision, not an oversight.
271
+ */
212
272
  export function attachmentOf(msg) {
213
273
  // Voice FIRST: a voice note also carries no photo and no document, and used
214
274
  // to fall through to `null` — so the message was dropped in silence, which is
@@ -220,20 +280,58 @@ export function attachmentOf(msg) {
220
280
  fileUniqueId: v.file_unique_id,
221
281
  kind: "voice",
222
282
  ...(v.file_size != null ? { fileSize: v.file_size } : {}),
283
+ ...(v.duration != null ? { durationSec: v.duration } : {}),
284
+ };
285
+ }
286
+ // A video, a GIF (`animation` — Telegram stores it as a soundless mp4) or a
287
+ // round "telescope" clip (`video_note`). `animation` comes BEFORE `document`
288
+ // on purpose: Telegram sends both fields for a GIF, and as a document it
289
+ // would lose its duration and its stills.
290
+ const mov = msg.video ?? msg.animation ?? msg.video_note;
291
+ if (mov?.file_id) {
292
+ return {
293
+ fileId: mov.file_id,
294
+ fileUniqueId: mov.file_unique_id,
295
+ kind: "video",
296
+ ...(mov.file_name ? { fileName: mov.file_name } : {}),
297
+ ...(mov.file_size != null ? { fileSize: mov.file_size } : {}),
298
+ ...(mov.mime_type ? { mimeType: mov.mime_type } : {}),
299
+ ...(mov.duration != null ? { durationSec: mov.duration } : {}),
223
300
  };
224
301
  }
225
302
  if (Array.isArray(msg.photo) && msg.photo.length) {
226
303
  const p = msg.photo[msg.photo.length - 1];
227
304
  return { fileId: p.file_id, fileUniqueId: p.file_unique_id, kind: "image", fileSize: p.file_size };
228
305
  }
306
+ // An audio FILE (a music track, a forwarded memo) — handed over as a file
307
+ // rather than transcribed: a 40-minute track would hold a turn hostage for
308
+ // the best part of an hour, and nobody shares an album for its lyrics. The
309
+ // point here is that it stops vanishing.
310
+ const a = msg.audio;
311
+ if (a?.file_id) {
312
+ return {
313
+ fileId: a.file_id,
314
+ fileUniqueId: a.file_unique_id,
315
+ kind: "file",
316
+ ...(a.file_name ? { fileName: a.file_name } : {}),
317
+ ...(a.file_size != null ? { fileSize: a.file_size } : {}),
318
+ ...(a.mime_type ? { mimeType: a.mime_type } : {}),
319
+ ...(a.duration != null ? { durationSec: a.duration } : {}),
320
+ };
321
+ }
229
322
  const d = msg.document;
230
323
  if (d?.file_id) {
231
324
  return {
232
325
  fileId: d.file_id,
233
326
  fileUniqueId: d.file_unique_id,
234
- kind: typeof d.mime_type === "string" && d.mime_type.startsWith("image/") ? "image" : "file",
327
+ kind: typeof d.mime_type === "string" && d.mime_type.startsWith("video/")
328
+ ? "video"
329
+ : typeof d.mime_type === "string" && d.mime_type.startsWith("image/")
330
+ ? "image"
331
+ : "file",
235
332
  ...(d.file_name ? { fileName: d.file_name } : {}),
236
333
  ...(d.file_size != null ? { fileSize: d.file_size } : {}),
334
+ ...(d.mime_type ? { mimeType: d.mime_type } : {}),
237
335
  };
238
336
  }
239
337
  return null;
@@ -283,6 +381,40 @@ export function pasteFileName(id, name, contentType) {
283
381
  const ext = pasteExtension(contentType);
284
382
  return base ? `paste-${id}-${base}.${ext}` : `paste-${id}.${ext}`;
285
383
  }
384
+ /**
385
+ * A WHITELIST from Telegram's mime type to an extension, for the media that
386
+ * arrives without a file name — a camera video has none.
387
+ *
388
+ * Same rule as `pasteExtension`: never split on "/", since the result lands in
389
+ * a filename. Unknown → null, and the caller decides what is honest.
390
+ */
391
+ const MEDIA_EXT = {
392
+ "video/mp4": "mp4",
393
+ "video/quicktime": "mov",
394
+ "video/webm": "webm",
395
+ "video/x-matroska": "mkv",
396
+ "video/mpeg": "mpeg",
397
+ "audio/mpeg": "mp3",
398
+ "audio/mp4": "m4a",
399
+ "audio/ogg": "ogg",
400
+ "audio/x-wav": "wav",
401
+ "audio/wav": "wav",
402
+ };
403
+ /** The extension a downloaded attachment should carry, or null when unknown.
404
+ * ffmpeg sniffs the container anyway, so this is for the humans and for the
405
+ * agent reading the path — being truthful matters more than being clever. */
406
+ export function mediaExtension(att) {
407
+ const fromMime = att.mimeType ? MEDIA_EXT[att.mimeType.split(";")[0].trim().toLowerCase()] : undefined;
408
+ if (fromMime)
409
+ return fromMime;
410
+ if (att.kind === "image")
411
+ return "jpg";
412
+ // A video Telegram transcoded itself is always mp4; saying nothing would
413
+ // leave the file extension-less, which reads as "unknown binary".
414
+ if (att.kind === "video")
415
+ return "mp4";
416
+ return null;
417
+ }
286
418
  /** Storage name under ~/.shadok-ai/media: keep the original name so Claude
287
419
  * has context, prefix with file_unique_id to avoid collisions, and strip
288
420
  * anything path-ish or shell-hostile. */
@@ -290,15 +422,56 @@ export function mediaFileName(att) {
290
422
  const base = att.fileName ? path.basename(att.fileName).replace(/[^\w.\- ]+/g, "_") : "";
291
423
  if (base)
292
424
  return `${att.fileUniqueId}-${base}`;
293
- return att.kind === "image" ? `${att.fileUniqueId}.jpg` : att.fileUniqueId;
425
+ const ext = mediaExtension(att);
426
+ return ext ? `${att.fileUniqueId}.${ext}` : att.fileUniqueId;
294
427
  }
295
- /** The prompt sent to the session for downloaded attachments: one absolute
296
- * path per line (Claude reads them itself — Read for images/PDF/text,
297
- * Bash for the rest), then the user's caption if any. */
428
+ /**
429
+ * The prompt sent to the session for downloaded attachments: one absolute path
430
+ * per line (Claude reads them itself — Read for images/PDF/text, Bash for the
431
+ * rest), then the user's caption if any.
432
+ *
433
+ * A video is the one that needs words around it. The path alone is a dead end
434
+ * (nothing in `Read` opens an mp4), so it comes with its stills — and with the
435
+ * sentence that says to READ them: a model told about a video and handed images
436
+ * next to it still answers "I can't watch videos" unless the link between the
437
+ * two is spelled out. Verified on exactly that failure.
438
+ */
298
439
  export function attachmentPrompt(items, caption) {
299
- const lines = items.map((i) => (i.kind === "image" ? `[Attached image: ${i.path}]` : `[Attached file: ${i.path}]`));
440
+ const lines = [];
441
+ for (const i of items) {
442
+ if (i.kind === "video") {
443
+ const len = i.durationSec ? ` (${clock(i.durationSec)})` : "";
444
+ lines.push(`[Attached video: ${i.path}${len}]`);
445
+ if (i.frames?.length) {
446
+ lines.push("Stills from it — read these images to see the video:");
447
+ i.frames.forEach((f, n) => lines.push(`[Attached image: ${f.path}] frame ${n + 1}/${i.frames.length} at ${clock(f.atSec)}`));
448
+ }
449
+ if (i.transcript?.trim())
450
+ lines.push(`⟦heard in the video⟧ "${i.transcript.trim()}"`);
451
+ else if (i.audioNote)
452
+ lines.push(`⟦video audio⟧ ${i.audioNote}`);
453
+ continue;
454
+ }
455
+ lines.push(i.kind === "image" ? `[Attached image: ${i.path}]` : `[Attached file: ${i.path}]`);
456
+ }
300
457
  return caption?.trim() ? lines.join("\n") + "\n" + caption : lines.join("\n");
301
458
  }
459
+ /** What a bot simply cannot fetch: the Bot API caps `getFile` at 20 MB, and no
460
+ * code on our side lifts it — so the notice names both sizes and the two
461
+ * things that do work. */
462
+ export function tooLargeNotice(att) {
463
+ const mb = att.fileSize ? `${(att.fileSize / 1_000_000).toFixed(1)} MB` : "this file";
464
+ const what = att.kind === "video" ? "video" : att.kind === "image" ? "image" : "file";
465
+ return `⚠️ Telegram only lets a bot download up to 20 MB, and this ${what} is ${mb}. Send a shorter clip, or compress it — a link to the file works too.`;
466
+ }
467
+ /** Was this download refused by the 20 MB ceiling rather than broken? The size
468
+ * we were told is the honest signal; Telegram's own wording is the fallback
469
+ * for a file it never sized. */
470
+ export function tooBigFor(att, err) {
471
+ if (att.fileSize && att.fileSize > TG_FILE_LIMIT)
472
+ return true;
473
+ return /too (big|large)/i.test(String(err?.message ?? err ?? ""));
474
+ }
302
475
  const htmlEscape = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
303
476
  /**
304
477
  * Render an agent's Markdown into the small HTML subset Telegram accepts
@@ -894,7 +1067,7 @@ export function startTelegram(port, authCookie) {
894
1067
  const text = b.preempting;
895
1068
  b.preempting = undefined;
896
1069
  clearTimeout(b.preemptTimer);
897
- promptTo(b, text, undefined, true);
1070
+ promptTo(b, text, b.lastSentFrom, true);
898
1071
  break;
899
1072
  }
900
1073
  b.typing.stop();
@@ -973,6 +1146,9 @@ export function startTelegram(port, authCookie) {
973
1146
  // name the author instead of showing an anonymous "pilot (elsewhere)".
974
1147
  const promptTo = (b, text, from, resend = false, voice = false) => {
975
1148
  b.lastSent = text;
1149
+ // A resend must keep the original author; only a NEW message sets one.
1150
+ if (!resend)
1151
+ b.lastSentFrom = from;
976
1152
  // Un message NEUF rouvre le droit d'interrompre ; un renvoi, non — sinon
977
1153
  // deux clients qui se disputent la session se relancent indéfiniment.
978
1154
  if (!resend)
@@ -1012,10 +1188,30 @@ export function startTelegram(port, authCookie) {
1012
1188
  // read audio, and a defensive skip is cheaper than finding out.
1013
1189
  if (i.att.kind === "voice")
1014
1190
  continue;
1015
- ok.push({ path: await downloadAttachment(i.att), kind: i.att.kind });
1191
+ const p = await downloadAttachment(i.att);
1192
+ const item = { path: p, kind: i.att.kind };
1193
+ if (i.att.kind === "video") {
1194
+ // Two stills per clip and no transcription: ten videos in one album
1195
+ // would otherwise mean sixty images and several minutes of whisper
1196
+ // before the agent is even woken.
1197
+ const look = await lookAtVideo(p, {
1198
+ durationSec: i.att.durationSec,
1199
+ maxFrames: ALBUM_FRAMES,
1200
+ transcribe: false,
1201
+ notify: (line) => void reply(b.chatId, b.threadId, line),
1202
+ });
1203
+ Object.assign(item, {
1204
+ ...(look.durationSec ? { durationSec: look.durationSec } : {}),
1205
+ ...(look.frames ? { frames: look.frames } : {}),
1206
+ ...(look.audioNote ? { audioNote: look.audioNote } : {}),
1207
+ });
1208
+ }
1209
+ ok.push(item);
1016
1210
  }
1017
1211
  catch (e) {
1018
- failed.push(`${i.att.fileName ?? i.att.fileUniqueId} (${e?.message ?? e})`);
1212
+ failed.push(tooBigFor(i.att, e)
1213
+ ? `${i.att.fileName ?? i.att.fileUniqueId} (over the 20 MB Telegram ceiling)`
1214
+ : `${i.att.fileName ?? i.att.fileUniqueId} (${e?.message ?? e})`);
1019
1215
  }
1020
1216
  }
1021
1217
  if (failed.length)
@@ -1405,11 +1601,38 @@ export function startTelegram(port, authCookie) {
1405
1601
  promptTo(b, caption ? `${out.text}\n\n${caption}` : out.text, senderName(msg.from), false, true);
1406
1602
  return;
1407
1603
  }
1408
- promptTo(b, attachmentPrompt([{ path: p, kind: att.kind }], caption), senderName(msg.from));
1604
+ const attQuote = replyContext(msg);
1605
+ // A video is not readable as a file: it is looked AT (stills) and
1606
+ // listened to (its audio track), both over the ffmpeg the voice
1607
+ // pipeline already provisions. Without this the agent received a path
1608
+ // to an mp4 and answered that it cannot watch videos.
1609
+ const item = { path: p, kind: att.kind };
1610
+ if (att.kind === "video") {
1611
+ const look = await lookAtVideo(p, {
1612
+ durationSec: att.durationSec,
1613
+ maxFrames: MAX_FRAMES,
1614
+ notify: (line) => void reply(chat.id, threadId, line),
1615
+ });
1616
+ if (look.trouble)
1617
+ await reply(chat.id, threadId, look.trouble);
1618
+ Object.assign(item, {
1619
+ ...(look.durationSec ? { durationSec: look.durationSec } : {}),
1620
+ ...(look.frames ? { frames: look.frames } : {}),
1621
+ ...(look.transcript ? { transcript: look.transcript } : {}),
1622
+ ...(look.audioNote ? { audioNote: look.audioNote } : {}),
1623
+ });
1624
+ }
1625
+ const attText = attachmentPrompt([item], caption);
1626
+ promptTo(b, attQuote ? attQuote + "\n" + attText : attText, senderName(msg.from));
1409
1627
  }
1410
1628
  catch (e) {
1411
1629
  b.typing.stop();
1412
- await reply(chat.id, threadId, `⚠️ could not download ${att.fileName ?? "the attachment"} (${e?.message ?? e})`);
1630
+ // The 20 MB ceiling is the common case for a phone video, and it is not
1631
+ // an error on anyone's side — so it gets its own actionable wording
1632
+ // instead of a raw message the user can do nothing with.
1633
+ await reply(chat.id, threadId, tooBigFor(att, e)
1634
+ ? tooLargeNotice(att)
1635
+ : `⚠️ could not download ${att.fileName ?? "the attachment"} (${e?.message ?? e})`);
1413
1636
  }
1414
1637
  return;
1415
1638
  }
@@ -1426,7 +1649,10 @@ export function startTelegram(port, authCookie) {
1426
1649
  b.pendingActions.push(wsMsg);
1427
1650
  return;
1428
1651
  }
1429
- promptTo(b, msg.text, senderName(msg.from));
1652
+ // A reply carries the message it answers: without it the agent reads
1653
+ // "fix that" with no trace of what "that" is.
1654
+ const quoted = replyContext(msg);
1655
+ promptTo(b, quoted ? quoted + "\n" + msg.text : msg.text, senderName(msg.from));
1430
1656
  };
1431
1657
  const handleCallback = async (cq) => {
1432
1658
  const action = parseCallback(cq.data ?? "");