whatsapp-pi 1.0.68 → 1.0.69

package/README.md

@@ -26,7 +26,7 @@ Pi is a powerful agentic AI coding assistant that operates in your terminal. Thi
 - **Group-Only Mode**: Bind the agent to a single WhatsApp group with `--whatsapp-group`
 - **Media Support**: 
   - **Vision Analysis**: Automatically forwards WhatsApp images to Pi for analysis.
-  - **Audio Transcription**: Transcribes voice notes when Whisper is installed.
+  - **Audio Transcription**: Transcribes voice notes locally with Whisper.cpp (`whisper-cpp-node`); `ffmpeg` is used to convert WhatsApp audio to 16 kHz mono WAV first.
   - **Document Handling**: Downloads and stores documents (PDF, text) for agent access; PDFs include a bounded text preview when readable.
 
 ## Prerequisites
@@ -59,11 +59,15 @@ See the [Pi documentation](https://pi.dev/docs/latest) for full setup, providers
 
 ### Audio Transcription
 
-To enable audio transcription features:
+Audio transcription uses `whisper-cpp-node` and `ffmpeg`.
+
+Install dependencies:
 ```bash
-python -m pip install -U openai-whisper
+npm install
 ```
 
+Make sure `ffmpeg` is available in PATH.
+
 PDF documents are parsed locally and do not require extra system utilities.
 If a PDF cannot be parsed automatically, it is still saved and forwarded with a clear fallback notice.
 
@@ -109,7 +113,7 @@ npm install
 pi -e whatsapp-pi.ts
 ```
 
-For verbose mode (shows Baileys trace logs for debugging):
+For verbose mode (shows Baileys trace logs and audio timing logs for debugging):
 ```bash
 pi -e whatsapp-pi.ts --verbose
 ```
@@ -168,8 +172,6 @@ The three read-only tools query the local recents store at `~/.pi/agent/extensio
 - **Add Group** - Add a WhatsApp group JID to the allowed groups list (format: 120363012345@g.us)
 - **Select a group** - Open a submenu with **History**, **Send Message**, **Print Group JID**, **Add Alias**, **Remove Alias**, **Remove Group**, and **Back**
 
-- **Back** - Return to main menu
-
 ### Recents Management
 - **History** - Open full message history for that conversation
 - **Send Message** - Send a new message without Pi suffix
@@ -187,7 +189,6 @@ Send these commands directly in WhatsApp to control the agent session:
 
 ```
 src/
-├── models/          # Type definitions
 ├── services/        # Core services (WhatsApp, Session, Recents, Media)
 └── ui/              # Menu handlers and TUI views
 
@@ -202,17 +203,10 @@ Run tests:
 npm test
 ```
 
-## Implementation Notes
-
-### Recent Feature Updates (2026-05)
+## Notes
 
-- **Auto-Connect Support**: Use the `--whatsapp-pi-online` flag to connect on startup when credentials already exist.
-- **Group-Only Mode**: Use `--whatsapp-group <jid>` to bind Pi to a single WhatsApp group. The group must also be present in Allowed Groups.
-- **Allowed Group Reaction Mode**: Each allowed group can be set to Active or Passive. Passive mode only replies when the bot is directly mentioned with @.
-- **Recents Store**: Recent conversations and message history are persisted in `~/.pi/agent/extensions/whatsapp-pi/recents/recents.json`.
-- **Message Detail / Reply**: Open a message from history to inspect full content and reply with `R`.
-- **Media Support**: Images are forwarded for vision analysis, audio is transcribed with Whisper, and PDFs are saved under `./.pi-data/whatsapp/documents/` with local text preview when available.
-- **Session Handling**: Saved state, allow list, and startup reconnects are restored automatically when available.
-- **Intelligent Message Filtering**: Messages ending with `π` are ignored to prevent bot loops.
-- **Storage Management**: Persistent data lives under `.pi-data/` plus the recents store in the user home directory.
-- **Improved Test Coverage (v1.0.59)**: Added unit tests for the `message_end` auto-reply handler, covering the happy path, disconnected guard, role guard, send failure, thrown exceptions, and the `send_wa_message` dedup flag. Fixed a Windows path separator bug in the recents service test suite.
+- `--whatsapp-pi-online` auto-connects when credentials already exist.
+- `--whatsapp-group <jid>` binds Pi to one WhatsApp group.
+- Media handling is local: images for vision, audio via Whisper.cpp + ffmpeg, documents stored under `.pi-data/whatsapp/documents/`.
+- Recents/history live in `~/.pi/agent/extensions/whatsapp-pi/recents/recents.json`.
+- Session state, allow lists, and startup reconnects are persisted locally.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "whatsapp-pi",
-  "version": "1.0.68",
+  "version": "1.0.69",
   "type": "module",
   "description": "WhatsApp integration extension for Pi",
   "main": "whatsapp-pi.ts",
@@ -33,9 +33,11 @@
   },
   "dependencies": {
     "@llamaindex/liteparse": "^1.5.3",
+    "@sinclair/typebox": "^0.27.10",
     "baileys": "^6.7.21",
     "pino": "^10.3.1",
-    "qrcode-terminal": "^0.12.0"
+    "qrcode-terminal": "^0.12.0",
+    "whisper-cpp-node": "^0.2.12"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.4",

package/src/i18n.ts

@@ -220,6 +220,13 @@ const fallback = {
     "incoming.media.documentDownloadFailed": "[WhatsApp-Pi] Failed to download document:",
     "incoming.media.documentDownloadFailedText": "[Document: {fileName} (download failed)]",
     "audio.emptyTranscription": "[Empty transcription]",
+    "audio.phase.download": "download",
+    "audio.phase.write": "write file",
+    "audio.phase.convert": "convert",
+    "audio.phase.whisper": "whisper",
+    "audio.phase.read": "read transcription",
+    "audio.phase.total": "total",
+    "audio.phaseTiming": "[WhatsApp-Pi] Audio {phase}: {duration} ms",
     "audio.transcriptionError": "[AudioService] Transcription error:",
     "audio.transcriptionErrorResult": "[Transcription error: {error}]",
     "baileys.filter.failedDecrypt": "Failed to decrypt message with any known session",
@@ -393,6 +400,13 @@ const translations: Record<Locale, Partial<Record<Key, string>>> = {
         "session.manager.failedDeleteAuthState": "Falha ao excluir estado de autenticação:",
         "incoming.media.audioTranscribing": "[WhatsApp-Pi] Transcrevendo áudio de {pushName}...",
         "incoming.media.audioTranscribed": "[Áudio transcrito]: {transcription}",
+        "audio.phase.download": "baixar",
+        "audio.phase.write": "salvar arquivo",
+        "audio.phase.convert": "converter",
+        "audio.phase.whisper": "transcrever",
+        "audio.phase.read": "ler transcrição",
+        "audio.phase.total": "total",
+        "audio.phaseTiming": "[WhatsApp-Pi] Áudio {phase}: {duration} ms",
         "incoming.media.imageDownloading": "[WhatsApp-Pi] Baixando imagem de {pushName}...",
         "incoming.media.imageDownloaded": "[WhatsApp-Pi] Imagem baixada. MIME: {imageMimeType} (original: {rawMime}), Tamanho: {size} bytes",
         "incoming.media.imageDownloadFailed": "[WhatsApp-Pi] Falha ao baixar imagem:",

package/src/services/audio.service.ts

@@ -1,90 +1,133 @@
-import { downloadContentFromMessage } from 'baileys';
-import { exec } from 'node:child_process';
-import { promisify } from 'node:util';
-import { writeFile, mkdir } from 'node:fs/promises';
-import { join } from 'node:path';
-import { existsSync } from 'node:fs';
-import { homedir } from 'node:os';
-import { createStoragePaths } from './storage-path.js';
-import { t } from '../i18n.js';
-
-const execAsync = promisify(exec);
-
-export class AudioService {
-    private readonly mediaDir = createStoragePaths().mediaDir;
-    private readonly whisperCommands = process.platform === 'win32'
-        ? ['whisper', 'py -m whisper', 'python -m whisper']
-        : [join(homedir(), '.local', 'bin', 'whisper'), 'whisper', 'python3 -m whisper', 'python -m whisper'];
-
-    constructor() {
-        if (!existsSync(this.mediaDir)) {
-            mkdir(this.mediaDir, { recursive: true }).catch(() => {});
-        }
-    }
-
-    async transcribe(audioMessage: any): Promise<string> {
-        try {
-            const filename = `audio_${Date.now()}`;
-            const inputPath = join(this.mediaDir, `${filename}.ogg`);
-
-            // Download audio content
-            const stream = await downloadContentFromMessage(audioMessage, 'audio');
-            let buffer = Buffer.from([]);
-            for await (const chunk of stream) {
-                buffer = Buffer.concat([buffer, chunk]);
-            }
-
-            await writeFile(inputPath, buffer);
-
-            // Transcribe using Whisper
-            // Using small model for better accuracy
-            await this.runWhisper(inputPath);
-
-            const txtPath = join(this.mediaDir, `${filename}.txt`);
-            if (existsSync(txtPath)) {
-                const fs = await import('node:fs/promises');
-                const text = await fs.readFile(txtPath, 'utf8');
-                return text.trim();
-            }
-
-            return t('audio.emptyTranscription');
-        } catch (error) {
-            console.error(t('audio.transcriptionError'), error);
-            return t('audio.transcriptionErrorResult', { error: error instanceof Error ? error.message : String(error) });
-        }
-    }
-
-    private async runWhisper(inputPath: string): Promise<void> {
-        const commandArgs = `"${inputPath}" --model small --language pt --output_format txt --output_dir "${this.mediaDir}" --fp16 False`;
-        let lastError: unknown;
-
-        for (const whisperCommand of this.whisperCommands) {
-            const command = `${whisperCommand} ${commandArgs}`;
-
-            try {
-                await execAsync(command);
-                return;
-            } catch (error) {
-                lastError = error;
-                if (!this.isMissingWhisperCommand(error)) {
-                    throw error;
-                }
-            }
-        }
-
-        throw lastError instanceof Error ? lastError : new Error(String(lastError));
-    }
-
-    private isMissingWhisperCommand(error: unknown): boolean {
-        if (!(error instanceof Error)) {
-            return false;
-        }
-
-        const anyError = error as Error & { code?: number | string; stderr?: string };
-        const message = `${anyError.message}\n${anyError.stderr ?? ''}`;
-
-        return anyError.code === 127
-            || anyError.code === 9009
-            || /not found|not recognized/i.test(message);
-    }
-}
+import { downloadContentFromMessage } from 'baileys';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { createStoragePaths } from './storage-path.js';
+import { WhatsAppPiLogger } from './whatsapp-pi.logger.js';
+import { tryCreateWhisperCppAudioTranscriber, type AudioTranscriber } from './whisper-cpp-audio.transcriber.js';
+import { t } from '../i18n.js';
+
+const execFileAsync = promisify(execFile);
+
+type AudioLogger = Pick<WhatsAppPiLogger, 'log' | 'error'>;
+type AudioPhase = 'download' | 'write' | 'convert' | 'whisper' | 'total';
+
+export class AudioService {
+    private readonly mediaDir = createStoragePaths().mediaDir;
+    private readonly logger: AudioLogger;
+    private readonly whisperCppTranscriber: AudioTranscriber | null;
+    private readonly ffmpegCommands = process.platform === 'win32' ? ['ffmpeg', 'ffmpeg.exe'] : ['ffmpeg'];
+
+    constructor(logger: AudioLogger = new WhatsAppPiLogger(false), whisperCppTranscriber?: AudioTranscriber | null) {
+        this.logger = logger;
+        this.whisperCppTranscriber = whisperCppTranscriber === undefined
+            ? tryCreateWhisperCppAudioTranscriber(logger)
+            : whisperCppTranscriber;
+
+        if (!existsSync(this.mediaDir)) {
+            mkdir(this.mediaDir, { recursive: true }).catch(() => {});
+        }
+    }
+
+    async transcribe(audioMessage: any): Promise<string> {
+        const totalStart = Date.now();
+
+        try {
+            const filename = `audio_${Date.now()}`;
+            const inputPath = join(this.mediaDir, `${filename}.ogg`);
+            const wavPath = join(this.mediaDir, `${filename}.wav`);
+
+            const buffer = await this.measurePhase('download', async () => {
+                const stream = await downloadContentFromMessage(audioMessage, 'audio');
+                let output = Buffer.from([]);
+
+                for await (const chunk of stream) {
+                    output = Buffer.concat([output, chunk]);
+                }
+
+                return output;
+            });
+
+            await this.measurePhase('write', async () => {
+                await writeFile(inputPath, buffer);
+            });
+
+            await this.measurePhase('convert', async () => {
+                await this.convertToWav(inputPath, wavPath);
+            });
+
+            const whisperCppTranscriber = this.whisperCppTranscriber;
+            if (!whisperCppTranscriber) {
+                throw new Error('whisper-cpp-node unavailable');
+            }
+
+            return await this.measurePhase('whisper', async () => {
+                const transcription = await whisperCppTranscriber.transcribe(wavPath);
+                const text = String(transcription ?? '').trim();
+                return text || t('audio.emptyTranscription');
+            });
+        } catch (error) {
+            console.error(t('audio.transcriptionError'), error);
+            return t('audio.transcriptionErrorResult', { error: error instanceof Error ? error.message : String(error) });
+        } finally {
+            this.logger.log(t('audio.phaseTiming', { phase: t('audio.phase.total'), duration: Date.now() - totalStart }));
+        }
+    }
+
+    private async measurePhase<T>(phase: Exclude<AudioPhase, 'total'>, action: () => Promise<T>): Promise<T> {
+        const start = Date.now();
+
+        try {
+            return await action();
+        } finally {
+            this.logger.log(t('audio.phaseTiming', { phase: this.getPhaseLabel(phase), duration: Date.now() - start }));
+        }
+    }
+
+    private getPhaseLabel(phase: Exclude<AudioPhase, 'total'>): string {
+        switch (phase) {
+            case 'download':
+                return t('audio.phase.download');
+            case 'write':
+                return t('audio.phase.write');
+            case 'convert':
+                return t('audio.phase.convert');
+            case 'whisper':
+                return t('audio.phase.whisper');
+        }
+    }
+
+    private async convertToWav(inputPath: string, outputPath: string): Promise<void> {
+        const args = ['-y', '-i', inputPath, '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le', outputPath];
+        let lastError: unknown;
+
+        for (const command of this.ffmpegCommands) {
+            try {
+                await execFileAsync(command, args, { windowsHide: true });
+                return;
+            } catch (error) {
+                lastError = error;
+                if (!this.isMissingFfmpegCommand(error)) {
+                    throw error;
+                }
+            }
+        }
+
+        throw lastError instanceof Error ? lastError : new Error('ffmpeg unavailable');
+    }
+
+    private isMissingFfmpegCommand(error: unknown): boolean {
+        if (!(error instanceof Error)) {
+            return false;
+        }
+
+        const anyError = error as Error & { code?: number | string; stderr?: string };
+        const message = `${anyError.message}\n${anyError.stderr ?? ''}`;
+
+        return anyError.code === 127
+            || anyError.code === 9009
+            || /not found|not recognized/i.test(message);
+    }
+}

package/src/services/whisper-cpp-audio.transcriber.ts

@@ -0,0 +1,179 @@
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+import { createWriteStream } from 'node:fs';
+import { mkdir, stat, unlink } from 'node:fs/promises';
+import https from 'node:https';
+import { createStoragePaths } from './storage-path.js';
+import type { WhatsAppPiLogger } from './whatsapp-pi.logger.js';
+
+type WhisperModule = {
+    createWhisperContext: (options: { model: string; use_gpu?: boolean; no_prints?: boolean }) => {
+        free?: () => void;
+    };
+    transcribeAsync: (context: { free?: () => void }, options: Record<string, unknown>) => Promise<{
+        segments?: Array<[string, string, string] | { text?: string }>;
+    }>;
+};
+
+type AudioLogger = Pick<WhatsAppPiLogger, 'log' | 'error'>;
+
+type WhisperContext = ReturnType<WhisperModule['createWhisperContext']>;
+type WhisperResult = Awaited<ReturnType<WhisperModule['transcribeAsync']>>;
+
+const DEFAULT_MODEL_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin';
+const MODEL_FILENAME = 'ggml-base.bin';
+
+let whisperModule: WhisperModule | undefined;
+let cachedContext: WhisperContext | undefined;
+let cachedModelPath: string | undefined;
+
+export interface AudioTranscriber {
+    transcribe(inputPath: string): Promise<string>;
+}
+
+function loadWhisperModule(): WhisperModule {
+    if (whisperModule) {
+        return whisperModule;
+    }
+
+    const require = createRequire(import.meta.url);
+    try {
+        whisperModule = require('whisper-cpp-node') as WhisperModule;
+        return whisperModule;
+    } catch {
+        throw new Error('whisper-cpp-node not installed. Run npm install.');
+    }
+}
+
+function getModelPath(): string {
+    const { root } = createStoragePaths();
+    return join(root, 'whisper', 'models', MODEL_FILENAME);
+}
+
+async function downloadFile(url: string, targetPath: string): Promise<void> {
+    await mkdir(dirname(targetPath), { recursive: true });
+    await unlink(targetPath).catch(() => undefined);
+
+    await new Promise<void>((resolve, reject) => {
+        const file = createWriteStream(targetPath);
+        const cleanup = () => void unlink(targetPath).catch(() => undefined);
+        const request = https.get(url, (response) => {
+            if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
+                const nextUrl = new URL(response.headers.location, url).toString();
+                response.resume();
+                file.close(cleanup);
+                downloadFile(nextUrl, targetPath).then(resolve).catch(reject);
+                return;
+            }
+
+            if (response.statusCode !== 200) {
+                file.close(cleanup);
+                reject(new Error(`Model download failed: HTTP ${response.statusCode}`));
+                return;
+            }
+
+            response.pipe(file);
+            file.once('finish', () => {
+                file.close((error) => {
+                    if (error) {
+                        cleanup();
+                        reject(error);
+                        return;
+                    }
+
+                    resolve();
+                });
+            });
+        });
+
+        request.on('error', (error) => {
+            file.close(cleanup);
+            reject(error);
+        });
+
+        file.on('error', (error) => {
+            request.destroy(error);
+            cleanup();
+            reject(error);
+        });
+    });
+}
+
+async function ensureWhisperModel(logger: AudioLogger): Promise<string> {
+    const modelPath = getModelPath();
+
+    try {
+        const stats = await stat(modelPath);
+        if (stats.size > 0) {
+            return modelPath;
+        }
+    } catch {
+        // download below
+    }
+
+    logger.log(`[WhatsApp-Pi] Whisper.cpp model download: ${modelPath}`);
+    await downloadFile(DEFAULT_MODEL_URL, modelPath);
+    return modelPath;
+}
+
+async function createContext(modelPath: string, logger: AudioLogger): Promise<WhisperContext> {
+    const { createWhisperContext } = loadWhisperModule();
+    logger.log('[WhatsApp-Pi] Whisper.cpp context init');
+    return createWhisperContext({
+        model: modelPath,
+        use_gpu: false,
+        no_prints: true
+    });
+}
+
+async function ensureContext(logger: AudioLogger): Promise<WhisperContext> {
+    const modelPath = await ensureWhisperModel(logger);
+    if (!cachedContext || cachedModelPath !== modelPath) {
+        cachedContext?.free?.();
+        cachedContext = await createContext(modelPath, logger);
+        cachedModelPath = modelPath;
+    }
+
+    return cachedContext;
+}
+
+function extractText(result: WhisperResult): string {
+    const segments = (result?.segments ?? []) as Array<[string, string, string] | { text?: string }>;
+    return segments
+        .map((segment) => Array.isArray(segment) ? segment[2] : segment.text)
+        .map((value) => String(value ?? '').trim())
+        .filter(Boolean)
+        .join(' ')
+        .trim();
+}
+
+export function tryCreateWhisperCppAudioTranscriber(logger: AudioLogger): AudioTranscriber | null {
+    try {
+        loadWhisperModule();
+    } catch {
+        return null;
+    }
+
+    return {
+        async transcribe(inputPath: string): Promise<string> {
+            const context = await ensureContext(logger);
+            const { transcribeAsync } = loadWhisperModule();
+            logger.log('[WhatsApp-Pi] Whisper.cpp transcribe');
+            const result = await transcribeAsync(context, {
+                fname_inp: inputPath,
+                language: 'pt',
+                no_timestamps: true,
+                no_context: false,
+                detect_language: false
+            });
+
+            return extractText(result);
+        }
+    };
+}
+
+export function freeWhisperCppContext() {
+    cachedContext?.free?.();
+    cachedContext = undefined;
+    cachedModelPath = undefined;
+}

package/whatsapp-pi.ts

@@ -43,8 +43,8 @@ export default function (pi: ExtensionAPI) {
     const sessionManager = new SessionManager();
     const whatsappService = new WhatsAppService(sessionManager);
     const recentsService = new RecentsService(sessionManager);
-    const audioService = new AudioService();
     const logger = new WhatsAppPiLogger(false);
+    const audioService = new AudioService(logger);
     const incomingMediaService = new IncomingMediaService(audioService, logger);
     const menuHandler = new MenuHandler(whatsappService, sessionManager, recentsService);
     let _ctx: ExtensionContext | undefined;
@@ -315,20 +315,21 @@ export default function (pi: ExtensionAPI) {
                 };
             }
 
-            const formattedMessage = params.message
+            const message = params.message ?? '';
+            const formattedMessage = message
                 .split('\n')
-                .map((line) => `    ${line}`)
+                .map((line: string) => `    ${line}`)
                 .join('\n');
 
-            console.log([
+            logger.log([
                 t("log.outgoing.title"),
-                t("log.outgoing.to", { jid: params.jid }),
+                t("log.outgoing.to", { jid: resolvedJid }),
                 t("log.outgoing.message"),
                 formattedMessage
             ].join('\n'));
 
             const outboundJid = whatsappService.resolveOutboundRecipientJid(resolvedJid);
-            const result = await whatsappService.sendMessage(outboundJid, params.message);
+            const result = await whatsappService.sendMessage(outboundJid, message);
 
             if (result.success) {
                 // Mark that tool already sent to this JID — prevents message_end from re-sending
@@ -336,20 +337,20 @@ export default function (pi: ExtensionAPI) {
                 await recentsService.recordMessage({
                     messageId: result.messageId!,
                     senderNumber: toRecentSenderNumber(outboundJid),
-                    text: params.message,
+                    text: message,
                     direction: 'outgoing',
                     timestamp: Date.now()
                 });
-                console.log([
+                logger.log([
                     t("log.result.title"),
-                    t("log.outgoing.to", { jid: params.jid }),
+                    t("log.outgoing.to", { jid: resolvedJid }),
                     t("log.result.status.sent"),
                     t("log.result.messageId", { messageId: result.messageId ?? t("log.unknownMessageId") })
                 ].join('\n'));
             } else {
-                console.log([
+                logger.log([
                     t("log.result.title"),
-                    t("log.outgoing.to", { jid: params.jid }),
+                    t("log.outgoing.to", { jid: resolvedJid }),
                     t("log.result.status.failed"),
                     t("log.result.error", { error: result.error ?? t("log.unknownError") })
                 ].join('\n'));
@@ -388,9 +389,9 @@ export default function (pi: ExtensionAPI) {
             // Create sender with the socket
             const sender = new ReactionSender(socket as any);
             const result = await sender.sendReaction({
-                jid: params.jid,
-                messageId: params.messageId,
-                emoji: params.emoji
+                jid: params.jid ?? '',
+                messageId: params.messageId ?? '',
+                emoji: params.emoji ?? ''
             });
 
             return {