sensorium-mcp 2.16.28 → 2.16.30

package/dist/tools/wait/drive-handler.js

@@ -0,0 +1,138 @@
+/**
+ * Drive activation and auto-consolidation logic extracted from wait-tool.ts.
+ *
+ * Contains:
+ *   - runAutoConsolidation(): 3 strategies (idle, episode-count, time-based)
+ *   - checkDriveActivation(): 3-phase autonomous drive (phase 1/2/3)
+ */
+import { formatDrivePrompt, PHASE3_APPROVAL_PROMPT } from "../../drive.js";
+import { log } from "../../logger.js";
+import { runIntelligentConsolidation } from "../../memory.js";
+import { getReminders } from "../../response-builders.js";
+import { backfillEmbeddings } from "../memory-tools.js";
+// ---------------------------------------------------------------------------
+// Auto-consolidation (3 strategies)
+// ---------------------------------------------------------------------------
+/**
+ * Checks three consolidation strategies and fires runIntelligentConsolidation
+ * + backfillEmbeddings as fire-and-forget when conditions are met.
+ *
+ * Strategies:
+ *   1. Idle-based  — 15 min idle, 30 min cooldown
+ *   2. Episode-count — ≥15 unconsolidated episodes, 30 min cooldown
+ *   3. Time-based  — every 4 hours regardless
+ */
+export function runAutoConsolidation(ctx) {
+    const { state, effectiveThreadId, getMemoryDb } = ctx;
+    // Strategy 1: Idle-based consolidation
+    try {
+        const idleMs = Date.now() - state.lastOperatorMessageAt;
+        if (idleMs > 15 * 60 * 1000 && effectiveThreadId !== undefined && Date.now() - state.lastConsolidationAt > 30 * 60 * 1000) {
+            state.lastConsolidationAt = Date.now();
+            const db = getMemoryDb();
+            void runIntelligentConsolidation(db, effectiveThreadId).then(async (report) => {
+                if (report.episodesProcessed > 0) {
+                    log.info(`[memory] Consolidation: ${report.episodesProcessed} episodes → ${report.notesCreated} notes`);
+                }
+                await backfillEmbeddings(db);
+            }).catch(err => {
+                log.error(`[memory] Consolidation error: ${err instanceof Error ? err.message : String(err)}`);
+            });
+        }
+    }
+    catch (_) { /* consolidation failure is non-fatal */ }
+    // Strategy 2: Episode-count consolidation — don't wait for idle.
+    // If many episodes accumulated during active use, consolidate now.
+    // This prevents stale/contradictory knowledge from persisting.
+    try {
+        if (effectiveThreadId !== undefined && Date.now() - state.lastConsolidationAt > 30 * 60 * 1000) {
+            const db = getMemoryDb();
+            const uncons = db.prepare("SELECT COUNT(*) as c FROM episodes WHERE consolidated = 0 AND thread_id = ?").get(effectiveThreadId);
+            if (uncons.c >= 15) {
+                state.lastConsolidationAt = Date.now();
+                void runIntelligentConsolidation(db, effectiveThreadId).then(async (report) => {
+                    if (report.episodesProcessed > 0) {
+                        log.info(`[memory] Episode-count consolidation: ${report.episodesProcessed} episodes → ${report.notesCreated} notes`);
+                    }
+                    await backfillEmbeddings(db);
+                }).catch(err => {
+                    log.error(`[memory] Episode-count consolidation error: ${err instanceof Error ? err.message : String(err)}`);
+                });
+            }
+        }
+    }
+    catch (_) { /* non-fatal */ }
+    // Strategy 3: Time-based consolidation — every 4 hours regardless.
+    // Ensures stale knowledge gets cleaned up even during low-activity periods.
+    try {
+        const TIME_CONSOLIDATION_INTERVAL = 4 * 60 * 60 * 1000; // 4 hours
+        if (effectiveThreadId !== undefined && Date.now() - state.lastConsolidationAt > TIME_CONSOLIDATION_INTERVAL) {
+            state.lastConsolidationAt = Date.now();
+            const db = getMemoryDb();
+            log.info(`[memory] Time-based consolidation triggered (4h since last)`);
+            void runIntelligentConsolidation(db, effectiveThreadId).then(async (report) => {
+                if (report.episodesProcessed > 0) {
+                    log.info(`[memory] Time-based consolidation: ${report.episodesProcessed} episodes → ${report.notesCreated} notes`);
+                }
+                await backfillEmbeddings(db);
+            }).catch(err => {
+                log.error(`[memory] Time-based consolidation error: ${err instanceof Error ? err.message : String(err)}`);
+            });
+        }
+    }
+    catch (_) { /* non-fatal */ }
+}
+// ---------------------------------------------------------------------------
+// 3-Phase Autonomous Drive
+// ---------------------------------------------------------------------------
+const DRIVE_COOLDOWN_MS = 30 * 60 * 1000; // 30 minutes between probability rolls
+/**
+ * Checks and handles the 3-phase autonomous drive logic.
+ *
+ * - Phase 3: If `drivePhase2Fired` → return PHASE3_APPROVAL_PROMPT
+ * - Phase 1+2: If idle > threshold and cooldown elapsed → formatDrivePrompt()
+ *              → if activated, set drivePhase2Fired=true, return drive prompt
+ *
+ * Returns a ToolResult if drive activated, or null if nothing to do.
+ */
+export function checkDriveActivation(ctx) {
+    const { state, effectiveThreadId, config, memoryRefresh, scheduleHint } = ctx;
+    const DRIVE_ACTIVATION_MS = config.DMN_ACTIVATION_HOURS * 60 * 60 * 1000;
+    const idleMs = Date.now() - state.lastOperatorMessageAt;
+    // Phase 3: If Phase 2 just fired and the agent came back without engaging
+    if (state.drivePhase2Fired) {
+        state.drivePhase2Fired = false; // Reset — only approve once
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: PHASE3_APPROVAL_PROMPT +
+                        memoryRefresh +
+                        scheduleHint +
+                        getReminders(effectiveThreadId, state.sessionStartedAt, config.AUTONOMOUS_MODE),
+                },
+            ],
+        };
+    }
+    // Phase 1: Probability gate (only if past threshold and cooldown elapsed)
+    if (idleMs >= DRIVE_ACTIVATION_MS && Date.now() - state.lastDriveAttemptAt >= DRIVE_COOLDOWN_MS) {
+        state.lastDriveAttemptAt = Date.now();
+        const driveResult = formatDrivePrompt(idleMs, config.DMN_ACTIVATION_HOURS);
+        if (driveResult.activated && driveResult.prompt) {
+            // Phase 2: Intention Elicitation — give the agent full autonomy
+            state.drivePhase2Fired = true;
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: driveResult.prompt,
+                    },
+                    ...(memoryRefresh ? [{ type: "text", text: memoryRefresh.replace(/^\n\n/, "") }] : []),
+                    { type: "text", text: scheduleHint + getReminders(effectiveThreadId, state.sessionStartedAt, config.AUTONOMOUS_MODE) },
+                ],
+            };
+        }
+    }
+    return null;
+}
+//# sourceMappingURL=drive-handler.js.map

package/dist/tools/wait/drive-handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"drive-handler.js","sourceRoot":"","sources":["../../../src/tools/wait/drive-handler.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EAAE,2BAA2B,EAAqB,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AA6BxD,8EAA8E;AAC9E,oCAAoC;AACpC,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAiB;IACpD,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC;IAEtD,uCAAuC;IACvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,qBAAqB,CAAC;QACxD,IAAI,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,IAAI,iBAAiB,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;YAC1H,KAAK,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;YACzB,KAAK,2BAA2B,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;gBAC1E,IAAI,MAAM,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;oBACjC,GAAG,CAAC,IAAI,CAAC,2BAA2B,MAAM,CAAC,iBAAiB,eAAe,MAAM,CAAC,YAAY,QAAQ,CAAC,CAAC;gBAC1G,CAAC;gBACD,MAAM,kBAAkB,CAAC,EAAE,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;gBACb,GAAG,CAAC,KAAK,CAAC,iCAAiC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjG,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,CAAC,wCAAwC,CAAC,CAAC;IAExD,iEAAiE;IACjE,mEAAmE;IACnE,+DAA+D;IAC/D,IAAI,CAAC;QACH,IAAI,iBAAiB,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;YAC/F,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,6EAA6E,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAkB,CAAC;YACjJ,IAAI,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,KAAK,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvC,KAAK,2BAA2B,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;oBAC1E,IAAI,MAAM,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;wBACjC,GAAG,CAAC,IAAI,CAAC,yCAAyC,MAAM,CAAC,iBAAiB,eAAe,MAAM,CAAC,YAAY,QAAQ,CAAC,CAAC;oBACxH,CAAC;oBACD,MAAM,kBAAkB,CAAC,EAAE,CAAC,CAAC;gBAC/B,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;oBACb,GAAG,CAAC,KAAK,CAAC,+CAA+C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC/G,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;IAE/B,mEAAmE;IACnE,4EAA4E;IAC5E,IAAI,CAAC;QACH,MAAM,2BAA2B,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU;QAClE,IAAI,iBAAiB,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,GAAG,2BAA2B,EAAE,CAAC;YAC5G,KAAK,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;YACzB,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;YACxE,KAAK,2BAA2B,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC,IAAI,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;gBAC1E,IAAI,MAAM,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;oBACjC,GAAG,CAAC,IAAI,CAAC,sCAAsC,MAAM,CAAC,iBAAiB,eAAe,MAAM,CAAC,YAAY,QAAQ,CAAC,CAAC;gBACrH,CAAC;gBACD,MAAM,kBAAkB,CAAC,EAAE,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;gBACb,GAAG,CAAC,KAAK,CAAC,4CAA4C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5G,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;AACjC,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,uCAAuC;AAEjF;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAiB;IACpD,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,GAAG,GAAG,CAAC;IAC9E,MAAM,mBAAmB,GAAG,MAAM,CAAC,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,qBAAqB,CAAC;IAExD,0EAA0E;IAC1E,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,CAAC,4BAA4B;QAC5D,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,sBAAsB;wBAC1B,aAAa;wBACb,YAAY;wBACZ,YAAY,CAAC,iBAAiB,EAAE,KAAK,CAAC,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC;iBAClF;aACF;SACF,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,IAAI,MAAM,IAAI,mBAAmB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,kBAAkB,IAAI,iBAAiB,EAAE,CAAC;QAChG,KAAK,CAAC,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAE3E,IAAI,WAAW,CAAC,SAAS,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;YAChD,gEAAgE;YAChE,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC9B,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,WAAW,CAAC,MAAM;qBACzB;oBACD,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,aAAa,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/F,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,YAAY,CAAC,iBAAiB,EAAE,KAAK,CAAC,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE;iBACvH;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/tools/wait/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Barrel re-export for the wait-tool subsystem.
+ *
+ * All wait-related modules live under src/tools/wait/.
+ * External consumers should import from this barrel.
+ */
+export { handleWaitForInstructions, type WaitToolContext, type WaitToolExtra } from "./poll-loop.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/wait/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/wait/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,yBAAyB,EAAE,KAAK,eAAe,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/tools/wait/index.js

@@ -0,0 +1,8 @@
+/**
+ * Barrel re-export for the wait-tool subsystem.
+ *
+ * All wait-related modules live under src/tools/wait/.
+ * External consumers should import from this barrel.
+ */
+export { handleWaitForInstructions } from "./poll-loop.js";
+//# sourceMappingURL=index.js.map

package/dist/tools/wait/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/tools/wait/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,yBAAyB,EAA4C,MAAM,gBAAgB,CAAC"}

package/dist/tools/wait/media-processor.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Media processing handlers extracted from wait-tool.ts (Phase 4).
+ *
+ * Handles voice messages, animations/GIFs, and video notes —
+ * downloading, transcribing, analyzing, and building content blocks.
+ */
+import type { StoredMessage } from "../../dispatcher.js";
+import { type Database } from "../../memory.js";
+import type { TelegramClient } from "../../telegram.js";
+export type ContentBlock = {
+    type: "text";
+    text: string;
+} | {
+    type: "image";
+    data: string;
+    mimeType: string;
+};
+/** Focused context that media processors need — not the full WaitToolContext. */
+export interface MediaContext {
+    telegram: TelegramClient;
+    openaiApiKey: string;
+    voiceAnalysisUrl: string;
+    effectiveThreadId: number;
+    sessionStartedAt: number;
+    getMemoryDb: () => Database;
+}
+/**
+ * Process a voice message: download → parallel transcription + VANPY →
+ * build tags → text block → save episode + voice signature.
+ *
+ * @returns Content blocks to append, and whether an episode was auto-saved.
+ */
+export declare function processVoice(msg: StoredMessage, ctx: MediaContext): Promise<{
+    blocks: ContentBlock[];
+    episodeSaved: boolean;
+}>;
+/**
+ * Process an animation/GIF: download → disk save → extractVideoFrames →
+ * analyzeVideoFrames → text block.
+ */
+export declare function processAnimation(msg: StoredMessage, ctx: MediaContext): Promise<ContentBlock[]>;
+/**
+ * Process a video note (circle video): download → parallel(frames, transcribe,
+ * VANPY) → vision analysis → tags → text block → save episode + voice signature.
+ *
+ * @returns Content blocks to append, and whether an episode was auto-saved.
+ */
+export declare function processVideoNote(msg: StoredMessage, ctx: MediaContext): Promise<{
+    blocks: ContentBlock[];
+    episodeSaved: boolean;
+}>;
+//# sourceMappingURL=media-processor.d.ts.map

package/dist/tools/wait/media-processor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"media-processor.d.ts","sourceRoot":"","sources":["../../../src/tools/wait/media-processor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAGL,KAAK,QAAQ,EACd,MAAM,iBAAiB,CAAC;AAQzB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOxD,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtD,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,cAAc,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,QAAQ,CAAC;CAC7B;AAMD;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,aAAa,EAClB,GAAG,EAAE,YAAY,GAChB,OAAO,CAAC;IAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAAC,YAAY,EAAE,OAAO,CAAA;CAAE,CAAC,CA6E5D;AAMD;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,aAAa,EAClB,GAAG,EAAE,YAAY,GAChB,OAAO,CAAC,YAAY,EAAE,CAAC,CAsDzB;AAMD;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,aAAa,EAClB,GAAG,EAAE,YAAY,GAChB,OAAO,CAAC;IAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAAC,YAAY,EAAE,OAAO,CAAA;CAAE,CAAC,CAkG5D"}

package/dist/tools/wait/media-processor.js

@@ -0,0 +1,261 @@
+/**
+ * Media processing handlers extracted from wait-tool.ts (Phase 4).
+ *
+ * Handles voice messages, animations/GIFs, and video notes —
+ * downloading, transcribing, analyzing, and building content blocks.
+ */
+import { saveFileToDisk } from "../../config.js";
+import { log } from "../../logger.js";
+import { saveEpisode, saveVoiceSignature, } from "../../memory.js";
+import { analyzeVideoFrames, analyzeVoiceEmotion, extractVideoFrames, transcribeAudio, } from "../../openai.js";
+import { buildAnalysisTags } from "../../response-builders.js";
+import { errorMessage } from "../../utils.js";
+// ---------------------------------------------------------------------------
+// Voice handler
+// ---------------------------------------------------------------------------
+/**
+ * Process a voice message: download → parallel transcription + VANPY →
+ * build tags → text block → save episode + voice signature.
+ *
+ * @returns Content blocks to append, and whether an episode was auto-saved.
+ */
+export async function processVoice(msg, ctx) {
+    const voice = msg.message.voice;
+    const blocks = [];
+    if (!ctx.openaiApiKey) {
+        blocks.push({
+            type: "text",
+            text: `[Voice message received — ${voice.duration}s — cannot transcribe: OPENAI_API_KEY not set]`,
+        });
+        return { blocks, episodeSaved: false };
+    }
+    try {
+        log.verbose("voice", `Downloading voice file ${voice.file_id}...`);
+        const { buffer } = await ctx.telegram.downloadFileAsBuffer(voice.file_id);
+        log.verbose("voice", `Downloaded ${buffer.length} bytes. Starting transcription + analysis...`);
+        // Run transcription and voice analysis in parallel.
+        const [transcript, analysis] = await Promise.all([
+            transcribeAudio(buffer, ctx.openaiApiKey),
+            ctx.voiceAnalysisUrl
+                ? analyzeVoiceEmotion(buffer, ctx.voiceAnalysisUrl)
+                : Promise.resolve(null),
+        ]);
+        // Build rich voice analysis tag from VANPY results.
+        const tags = buildAnalysisTags(analysis);
+        const analysisTag = tags.length > 0 ? ` | ${tags.join(", ")}` : "";
+        blocks.push({
+            type: "text",
+            text: transcript
+                ? `[Voice message — ${voice.duration}s${analysisTag}, transcribed]: ${transcript}`
+                : `[Voice message — ${voice.duration}s${analysisTag}, transcribed]: (empty — no speech detected)`,
+        });
+        // Auto-save voice signature
+        let episodeSaved = false;
+        if (analysis && ctx.effectiveThreadId !== undefined) {
+            try {
+                const db = ctx.getMemoryDb();
+                const sessionId = `session_${ctx.sessionStartedAt}`;
+                const epId = saveEpisode(db, {
+                    sessionId,
+                    threadId: ctx.effectiveThreadId,
+                    type: "operator_message",
+                    modality: "voice",
+                    content: { text: transcript ?? "", duration: voice.duration },
+                    importance: 0.6,
+                });
+                saveVoiceSignature(db, {
+                    episodeId: epId,
+                    emotion: analysis.emotion ?? undefined,
+                    arousal: analysis.arousal ?? undefined,
+                    dominance: analysis.dominance ?? undefined,
+                    valence: analysis.valence ?? undefined,
+                    speechRate: analysis.paralinguistics?.speech_rate ?? undefined,
+                    meanPitchHz: analysis.paralinguistics?.mean_pitch_hz ?? undefined,
+                    pitchStdHz: analysis.paralinguistics?.pitch_std_hz ?? undefined,
+                    jitter: analysis.paralinguistics?.jitter ?? undefined,
+                    shimmer: analysis.paralinguistics?.shimmer ?? undefined,
+                    hnrDb: analysis.paralinguistics?.hnr_db ?? undefined,
+                    audioEvents: analysis.audio_events?.map(e => ({ label: e.label, confidence: e.score })),
+                    durationSec: voice.duration,
+                });
+                episodeSaved = true;
+            }
+            catch (_) { /* non-fatal */ }
+        }
+        return { blocks, episodeSaved };
+    }
+    catch (err) {
+        blocks.push({
+            type: "text",
+            text: `[Voice message — ${voice.duration}s — transcription failed: ${errorMessage(err)}]`,
+        });
+        return { blocks, episodeSaved: false };
+    }
+}
+// ---------------------------------------------------------------------------
+// Animation / GIF handler
+// ---------------------------------------------------------------------------
+/**
+ * Process an animation/GIF: download → disk save → extractVideoFrames →
+ * analyzeVideoFrames → text block.
+ */
+export async function processAnimation(msg, ctx) {
+    const anim = msg.message.animation;
+    const animDuration = anim.duration ?? 3; // default to 3s if Telegram omits duration
+    const blocks = [];
+    if (!ctx.openaiApiKey) {
+        blocks.push({
+            type: "text",
+            text: `(The operator sent a GIF — cannot analyze: OPENAI_API_KEY not set)`,
+        });
+        return blocks;
+    }
+    try {
+        log.verbose("gif", `Downloading animation ${anim.file_id} (~${animDuration}s)...`);
+        const { buffer } = await ctx.telegram.downloadFileAsBuffer(anim.file_id);
+        const diskPath = saveFileToDisk(buffer, "gif-animation.mp4");
+        log.verbose("gif", `Downloaded ${buffer.length} bytes. Extracting frames...`);
+        // Extract frames with ffmpeg (same as video_notes).
+        const frames = await extractVideoFrames(buffer, animDuration).catch((err) => {
+            log.error(`[gif] Frame extraction failed: ${errorMessage(err)}`);
+            return [];
+        });
+        // Analyze frames with GPT-4o-mini vision (same as video_notes).
+        let sceneDescription = null;
+        if (frames.length > 0) {
+            try {
+                log.verbose("gif", `Analyzing ${frames.length} frames with GPT-4o-mini vision...`);
+                sceneDescription = await analyzeVideoFrames(frames, animDuration, ctx.openaiApiKey);
+                log.verbose("gif", `Vision analysis complete.`);
+            }
+            catch (visionErr) {
+                log.error(`[gif] Vision analysis failed: ${visionErr}`);
+                sceneDescription = null;
+            }
+        }
+        const caption = msg.message.caption || "";
+        const parts = [];
+        parts.push(`(The operator sent a GIF — ${animDuration}s)`);
+        if (sceneDescription)
+            parts.push(`Scene: ${sceneDescription}`);
+        if (!sceneDescription)
+            parts.push("(no visual content could be extracted)");
+        parts.push(`Saved to: ${diskPath}`);
+        if (caption)
+            parts.push(`Caption: ${caption}`);
+        blocks.push({ type: "text", text: parts.join("\n") });
+    }
+    catch (err) {
+        blocks.push({
+            type: "text",
+            text: `(The operator sent a GIF — analysis failed: ${errorMessage(err)})`,
+        });
+    }
+    return blocks;
+}
+// ---------------------------------------------------------------------------
+// Video note handler
+// ---------------------------------------------------------------------------
+/**
+ * Process a video note (circle video): download → parallel(frames, transcribe,
+ * VANPY) → vision analysis → tags → text block → save episode + voice signature.
+ *
+ * @returns Content blocks to append, and whether an episode was auto-saved.
+ */
+export async function processVideoNote(msg, ctx) {
+    const vn = msg.message.video_note;
+    const blocks = [];
+    if (!ctx.openaiApiKey) {
+        blocks.push({
+            type: "text",
+            text: `[Video note received — ${vn.duration}s — cannot analyze: OPENAI_API_KEY not set]`,
+        });
+        return { blocks, episodeSaved: false };
+    }
+    try {
+        log.verbose("video-note", `Downloading circle video ${vn.file_id} (${vn.duration}s)...`);
+        const { buffer } = await ctx.telegram.downloadFileAsBuffer(vn.file_id);
+        log.verbose("video-note", `Downloaded ${buffer.length} bytes. Extracting frames + transcribing...`);
+        // Run frame extraction, audio transcription, and voice analysis in parallel.
+        const [frames, transcript, analysis] = await Promise.all([
+            extractVideoFrames(buffer, vn.duration).catch((err) => {
+                log.error(`[video-note] Frame extraction failed: ${errorMessage(err)}`);
+                return [];
+            }),
+            transcribeAudio(buffer, ctx.openaiApiKey, "video.mp4").catch(() => ""),
+            ctx.voiceAnalysisUrl
+                ? analyzeVoiceEmotion(buffer, ctx.voiceAnalysisUrl, {
+                    mimeType: "video/mp4",
+                    filename: "video.mp4",
+                }).catch(() => null)
+                : Promise.resolve(null),
+        ]);
+        // Analyze frames with GPT-4o-mini vision.
+        let sceneDescription = "";
+        if (frames.length > 0) {
+            try {
+                log.verbose("video-note", `Analyzing ${frames.length} frames with GPT-4o-mini vision...`);
+                sceneDescription = await analyzeVideoFrames(frames, vn.duration, ctx.openaiApiKey);
+                log.verbose("video-note", `Vision analysis complete.`);
+            }
+            catch (visionErr) {
+                log.error(`[video-note] Vision analysis failed: ${visionErr}`);
+                sceneDescription = null;
+            }
+        }
+        // Build analysis tags (same as voice messages).
+        const tags = buildAnalysisTags(analysis);
+        const analysisTag = tags.length > 0 ? ` | ${tags.join(", ")}` : "";
+        const parts = [];
+        parts.push(`[Video note — ${vn.duration}s${analysisTag}]`);
+        if (sceneDescription)
+            parts.push(`Scene: ${sceneDescription}`);
+        if (transcript)
+            parts.push(`Audio: "${transcript}"`);
+        if (!sceneDescription && !transcript)
+            parts.push("(no visual or audio content could be extracted)");
+        blocks.push({ type: "text", text: parts.join("\n") });
+        // Auto-save voice signature for video notes
+        let episodeSaved = false;
+        if (analysis && ctx.effectiveThreadId !== undefined) {
+            try {
+                const db = ctx.getMemoryDb();
+                const sessionId = `session_${ctx.sessionStartedAt}`;
+                const epId = saveEpisode(db, {
+                    sessionId,
+                    threadId: ctx.effectiveThreadId,
+                    type: "operator_message",
+                    modality: "video_note",
+                    content: { text: transcript ?? "", scene: sceneDescription ?? "", duration: vn.duration },
+                    importance: 0.6,
+                });
+                saveVoiceSignature(db, {
+                    episodeId: epId,
+                    emotion: analysis.emotion ?? undefined,
+                    arousal: analysis.arousal ?? undefined,
+                    dominance: analysis.dominance ?? undefined,
+                    valence: analysis.valence ?? undefined,
+                    speechRate: analysis.paralinguistics?.speech_rate ?? undefined,
+                    meanPitchHz: analysis.paralinguistics?.mean_pitch_hz ?? undefined,
+                    pitchStdHz: analysis.paralinguistics?.pitch_std_hz ?? undefined,
+                    jitter: analysis.paralinguistics?.jitter ?? undefined,
+                    shimmer: analysis.paralinguistics?.shimmer ?? undefined,
+                    hnrDb: analysis.paralinguistics?.hnr_db ?? undefined,
+                    audioEvents: analysis.audio_events?.map(e => ({ label: e.label, confidence: e.score })),
+                    durationSec: vn.duration,
+                });
+                episodeSaved = true;
+            }
+            catch (_) { /* non-fatal */ }
+        }
+        return { blocks, episodeSaved };
+    }
+    catch (err) {
+        blocks.push({
+            type: "text",
+            text: `[Video note — ${vn.duration}s — analysis failed: ${errorMessage(err)}]`,
+        });
+        return { blocks, episodeSaved: false };
+    }
+}
+//# sourceMappingURL=media-processor.js.map

package/dist/tools/wait/media-processor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"media-processor.js","sourceRoot":"","sources":["../../../src/tools/wait/media-processor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACtC,OAAO,EACL,WAAW,EACX,kBAAkB,GAEnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,EAClB,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAoB9C,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAkB,EAClB,GAAiB;IAEjB,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,KAAM,CAAC;IACjC,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,6BAA6B,KAAK,CAAC,QAAQ,gDAAgD;SAClG,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC;QACH,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,0BAA0B,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;QACnE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1E,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,cAAc,MAAM,CAAC,MAAM,8CAA8C,CAAC,CAAC;QAEhG,oDAAoD;QACpD,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC/C,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,YAAY,CAAC;YACzC,GAAG,CAAC,gBAAgB;gBAClB,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAAE,GAAG,CAAC,gBAAgB,CAAC;gBACnD,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;SAC1B,CAAC,CAAC;QAEH,oDAAoD;QACpD,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEnE,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,UAAU;gBACd,CAAC,CAAC,oBAAoB,KAAK,CAAC,QAAQ,IAAI,WAAW,mBAAmB,UAAU,EAAE;gBAClF,CAAC,CAAC,oBAAoB,KAAK,CAAC,QAAQ,IAAI,WAAW,8CAA8C;SACpG,CAAC,CAAC;QAEH,4BAA4B;QAC5B,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,QAAQ,IAAI,GAAG,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,WAAW,GAAG,CAAC,gBAAgB,EAAE,CAAC;gBACpD,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,EAAE;oBAC3B,SAAS;oBACT,QAAQ,EAAE,GAAG,CAAC,iBAAiB;oBAC/B,IAAI,EAAE,kBAAkB;oBACxB,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE;oBAC7D,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;gBACH,kBAAkB,CAAC,EAAE,EAAE;oBACrB,SAAS,EAAE,IAAI;oBACf,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,SAAS;oBACtC,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,SAAS;oBACtC,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,SAAS;oBAC1C,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,SAAS;oBACtC,UAAU,EAAE,QAAQ,CAAC,eAAe,EAAE,WAAW,IAAI,SAAS;oBAC9D,WAAW,EAAE,QAAQ,CAAC,eAAe,EAAE,aAAa,IAAI,SAAS;oBACjE,UAAU,EAAE,QAAQ,CAAC,eAAe,EAAE,YAAY,IAAI,SAAS;oBAC/D,MAAM,EAAE,QAAQ,CAAC,eAAe,EAAE,MAAM,IAAI,SAAS;oBACrD,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,OAAO,IAAI,SAAS;oBACvD,KAAK,EAAE,QAAQ,CAAC,eAAe,EAAE,MAAM,IAAI,SAAS;oBACpD,WAAW,EAAE,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;oBACvF,WAAW,EAAE,KAAK,CAAC,QAAQ;iBAC5B,CAAC,CAAC;gBACH,YAAY,GAAG,IAAI,CAAC;YACtB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,oBAAoB,KAAK,CAAC,QAAQ,6BAA6B,YAAY,CAAC,GAAG,CAAC,GAAG;SAC1F,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAkB,EAClB,GAAiB;IAEjB,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,SAAU,CAAC;IACpC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,2CAA2C;IACpF,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,oEAAoE;SAC3E,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,yBAAyB,IAAI,CAAC,OAAO,MAAM,YAAY,OAAO,CAAC,CAAC;QACnF,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;QAC7D,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,cAAc,MAAM,CAAC,MAAM,8BAA8B,CAAC,CAAC;QAE9E,oDAAoD;QACpD,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1E,GAAG,CAAC,KAAK,CAAC,kCAAkC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjE,OAAO,EAAc,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,gEAAgE;QAChE,IAAI,gBAAgB,GAAkB,IAAI,CAAC;QAC3C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,MAAM,CAAC,MAAM,oCAAoC,CAAC,CAAC;gBACnF,gBAAgB,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;gBACpF,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;YAClD,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,GAAG,CAAC,KAAK,CAAC,iCAAiC,SAAS,EAAE,CAAC,CAAC;gBACxD,gBAAgB,GAAG,IAAI,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,8BAA8B,YAAY,IAAI,CAAC,CAAC;QAC3D,IAAI,gBAAgB;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,gBAAgB,EAAE,CAAC,CAAC;QAC/D,IAAI,CAAC,gBAAgB;YAAE,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QAC5E,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QACpC,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,+CAA+C,YAAY,CAAC,GAAG,CAAC,GAAG;SAC1E,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAkB,EAClB,GAAiB;IAEjB,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,UAAW,CAAC;IACnC,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,0BAA0B,EAAE,CAAC,QAAQ,6CAA6C;SACzF,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC;QACH,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,4BAA4B,EAAE,CAAC,OAAO,KAAK,EAAE,CAAC,QAAQ,OAAO,CAAC,CAAC;QACzF,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;QACvE,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,cAAc,MAAM,CAAC,MAAM,6CAA6C,CAAC,CAAC;QAEpG,6EAA6E;QAC7E,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvD,kBAAkB,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACpD,GAAG,CAAC,KAAK,CAAC,yCAAyC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACxE,OAAO,EAAc,CAAC;YACxB,CAAC,CAAC;YACF,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,gBAAgB;gBAClB,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAAE,GAAG,CAAC,gBAAgB,EAAE;oBAChD,QAAQ,EAAE,WAAW;oBACrB,QAAQ,EAAE,WAAW;iBACtB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;gBACtB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;SAC1B,CAAC,CAAC;QAEH,0CAA0C;QAC1C,IAAI,gBAAgB,GAAkB,EAAE,CAAC;QACzC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,aAAa,MAAM,CAAC,MAAM,oCAAoC,CAAC,CAAC;gBAC1F,gBAAgB,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;gBACnF,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,2BAA2B,CAAC,CAAC;YACzD,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,GAAG,CAAC,KAAK,CAAC,wCAAwC,SAAS,EAAE,CAAC,CAAC;gBAC/D,gBAAgB,GAAG,IAAI,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,gDAAgD;QAChD,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEnE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,QAAQ,IAAI,WAAW,GAAG,CAAC,CAAC;QAC3D,IAAI,gBAAgB;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,gBAAgB,EAAE,CAAC,CAAC;QAC/D,IAAI,UAAU;YAAE,KAAK,CAAC,IAAI,CAAC,WAAW,UAAU,GAAG,CAAC,CAAC;QACrD,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU;YAAE,KAAK,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAEpG,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEtD,4CAA4C;QAC5C,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,QAAQ,IAAI,GAAG,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,WAAW,GAAG,CAAC,gBAAgB,EAAE,CAAC;gBACpD,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,EAAE;oBAC3B,SAAS;oBACT,QAAQ,EAAE,GAAG,CAAC,iBAAiB;oBAC/B,IAAI,EAAE,kBAAkB;oBACxB,QAAQ,EAAE,YAAY;oBACtB,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,IAAI,EAAE,EAAE,KAAK,EAAE,gBAAgB,IAAI,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE;oBACzF,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;gBACH,kBAAkB,CAAC,EAAE,EAAE;oBACrB,SAAS,EAAE,IAAI;oBACf,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,SAAS;oBACtC,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,SAAS;oBACtC,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,SAAS;oBAC1C,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,SAAS;oBACtC,UAAU,EAAE,QAAQ,CAAC,eAAe,EAAE,WAAW,IAAI,SAAS;oBAC9D,WAAW,EAAE,QAAQ,CAAC,eAAe,EAAE,aAAa,IAAI,SAAS;oBACjE,UAAU,EAAE,QAAQ,CAAC,eAAe,EAAE,YAAY,IAAI,SAAS;oBAC/D,MAAM,EAAE,QAAQ,CAAC,eAAe,EAAE,MAAM,IAAI,SAAS;oBACrD,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,OAAO,IAAI,SAAS;oBACvD,KAAK,EAAE,QAAQ,CAAC,eAAe,EAAE,MAAM,IAAI,SAAS;oBACpD,WAAW,EAAE,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;oBACvF,WAAW,EAAE,EAAE,CAAC,QAAQ;iBACzB,CAAC,CAAC;gBACH,YAAY,GAAG,IAAI,CAAC;YACtB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,iBAAiB,EAAE,CAAC,QAAQ,wBAAwB,YAAY,CAAC,GAAG,CAAC,GAAG;SAC/E,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;IACzC,CAAC;AACH,CAAC"}

package/dist/tools/wait/message-delivery.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Message delivery logic extracted from wait-tool.ts (Phase 4).
+ *
+ * Handles:
+ *   - Simple message processing (photo, document, text, sticker)
+ *   - Auto-ingesting episodes for messages not saved by media handlers
+ *   - Smart context injection (embedding search + GPT-4o-mini filter)
+ *   - Intent classification and final operator response assembly
+ */
+import type { StoredMessage } from "../../dispatcher.js";
+import { type initMemoryDb } from "../../memory.js";
+import type { TelegramClient } from "../../telegram.js";
+type TextBlock = {
+    type: "text";
+    text: string;
+};
+type ImageBlock = {
+    type: "image";
+    data: string;
+    mimeType: string;
+};
+type ContentBlock = TextBlock | ImageBlock;
+type ToolResult = {
+    content: Array<ContentBlock>;
+    isError?: boolean;
+};
+export interface MessageDeliveryContext {
+    telegram: TelegramClient;
+    getMemoryDb: () => ReturnType<typeof initMemoryDb>;
+    effectiveThreadId: number;
+    sessionStartedAt: number;
+    autonomousMode: boolean;
+}
+/**
+ * Process non-media message types for a single StoredMessage.
+ * Handles: photo, document, text, sticker.
+ * Returns content blocks for the processed parts.
+ */
+export declare function processSimpleMessage(msg: StoredMessage, telegram: TelegramClient): Promise<ContentBlock[]>;
+/**
+ * If contentBlocks is empty after processing all messages, push an
+ * unsupported-type placeholder and log the message fields for debugging.
+ */
+export declare function handleEmptyContent(contentBlocks: ContentBlock[], stored: StoredMessage[]): void;
+/**
+ * Auto-ingest episodes for messages that were not already saved by
+ * voice/video handlers (identified by savedEpisodeUpdateIds).
+ */
+export declare function autoIngestEpisodes(stored: StoredMessage[], savedEpisodeUpdateIds: Set<number>, ctx: Pick<MessageDeliveryContext, "getMemoryDb" | "effectiveThreadId" | "sessionStartedAt">): void;
+/**
+ * Smart context injection: retrieves candidate memory notes via embedding
+ * search, then uses GPT-4o-mini to select ONLY the notes truly relevant
+ * to the operator's message. Falls back to keyword search if embedding
+ * fails, and to raw top-3 if the LLM filter fails.
+ */
+export declare function buildSmartContext(operatorText: string, ctx: Pick<MessageDeliveryContext, "getMemoryDb" | "effectiveThreadId">): Promise<string>;
+/**
+ * Build the final operator message response: intent classification,
+ * reminder selection, and the `<<< OPERATOR MESSAGE >>>` envelope.
+ */
+export declare function assembleOperatorResponse(contentBlocks: ContentBlock[], operatorText: string, hasVoiceMessages: boolean, autoMemoryContext: string, ctx: Pick<MessageDeliveryContext, "effectiveThreadId" | "sessionStartedAt" | "autonomousMode">): ToolResult;
+export {};
+//# sourceMappingURL=message-delivery.d.ts.map

package/dist/tools/wait/message-delivery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"message-delivery.d.ts","sourceRoot":"","sources":["../../../src/tools/wait/message-delivery.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,iBAAiB,CAAC;AAMzB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOxD,KAAK,SAAS,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAChD,KAAK,UAAU,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AACpE,KAAK,YAAY,GAAG,SAAS,GAAG,UAAU,CAAC;AAC3C,KAAK,UAAU,GAAG;IAAE,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAEtE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,cAAc,CAAC;IACzB,WAAW,EAAE,MAAM,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;IACnD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,OAAO,CAAC;CACzB;AAMD;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,aAAa,EAClB,QAAQ,EAAE,cAAc,GACvB,OAAO,CAAC,YAAY,EAAE,CAAC,CAkFzB;AAMD;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,aAAa,EAAE,YAAY,EAAE,EAC7B,MAAM,EAAE,aAAa,EAAE,GACtB,IAAI,CAWN;AAMD;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,aAAa,EAAE,EACvB,qBAAqB,EAAE,GAAG,CAAC,MAAM,CAAC,EAClC,GAAG,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,GAAG,mBAAmB,GAAG,kBAAkB,CAAC,GAC1F,IAAI,CA0BN;AAMD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,GAAG,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,GAAG,mBAAmB,CAAC,GACrE,OAAO,CAAC,MAAM,CAAC,CAsFjB;AAMD;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,YAAY,EAAE,EAC7B,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,OAAO,EACzB,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,IAAI,CAAC,sBAAsB,EAAE,mBAAmB,GAAG,kBAAkB,GAAG,gBAAgB,CAAC,GAC7F,UAAU,CA4BZ"}