shuvmaki 0.4.26

package/dist/genai-worker.js

@@ -0,0 +1,297 @@
+// Worker thread for GenAI voice processing.
+// Runs in a separate thread to handle audio encoding/decoding without blocking.
+// Resamples 24kHz GenAI output to 48kHz stereo Opus packets for Discord.
+import { parentPort, threadId } from 'node:worker_threads';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { Resampler } from '@purinton/resampler';
+import * as prism from 'prism-media';
+import { startGenAiSession } from './genai.js';
+import { getTools } from './tools.js';
+import { createLogger } from './logger.js';
+if (!parentPort) {
+    throw new Error('This module must be run as a worker thread');
+}
+const workerLogger = createLogger(`WORKER ${threadId}`);
+workerLogger.log('GenAI worker started');
+// Define sendError early so it can be used by global handlers
+function sendError(error) {
+    if (parentPort) {
+        parentPort.postMessage({
+            type: 'error',
+            error,
+        });
+    }
+}
+// Add global error handlers for the worker thread
+process.on('uncaughtException', (error) => {
+    workerLogger.error('Uncaught exception in worker:', error);
+    sendError(`Worker crashed: ${error.message}`);
+    // Exit immediately on uncaught exception
+    process.exit(1);
+});
+process.on('unhandledRejection', (reason, promise) => {
+    workerLogger.error('Unhandled rejection in worker:', reason, 'at promise:', promise);
+    sendError(`Worker unhandled rejection: ${reason}`);
+});
+// Audio configuration
+const AUDIO_CONFIG = {
+    inputSampleRate: 24000, // GenAI output
+    inputChannels: 1,
+    outputSampleRate: 48000, // Discord expects
+    outputChannels: 2,
+    opusFrameSize: 960, // 20ms at 48kHz
+};
+// Initialize audio processing components
+const resampler = new Resampler({
+    inRate: AUDIO_CONFIG.inputSampleRate,
+    outRate: AUDIO_CONFIG.outputSampleRate,
+    inChannels: AUDIO_CONFIG.inputChannels,
+    outChannels: AUDIO_CONFIG.outputChannels,
+    volume: 1,
+    filterWindow: 8,
+});
+const opusEncoder = new prism.opus.Encoder({
+    rate: AUDIO_CONFIG.outputSampleRate,
+    channels: AUDIO_CONFIG.outputChannels,
+    frameSize: AUDIO_CONFIG.opusFrameSize,
+});
+// Pipe resampler to encoder with error handling
+resampler.pipe(opusEncoder).on('error', (error) => {
+    workerLogger.error('Pipe error between resampler and encoder:', error);
+    sendError(`Audio pipeline error: ${error.message}`);
+});
+// Opus packet queue and interval for 20ms packet sending
+const opusPacketQueue = [];
+let packetInterval = null;
+// Send packets every 20ms
+function startPacketSending() {
+    if (packetInterval)
+        return;
+    packetInterval = setInterval(() => {
+        const packet = opusPacketQueue.shift();
+        if (!packet)
+            return;
+        // Transfer packet as ArrayBuffer
+        const arrayBuffer = packet.buffer.slice(packet.byteOffset, packet.byteOffset + packet.byteLength);
+        parentPort.postMessage({
+            type: 'assistantOpusPacket',
+            packet: arrayBuffer,
+        }, [arrayBuffer]);
+    }, 20);
+}
+function stopPacketSending() {
+    if (packetInterval) {
+        clearInterval(packetInterval);
+        packetInterval = null;
+    }
+    opusPacketQueue.length = 0;
+}
+// Session state
+let session = null;
+// Audio log stream for assistant audio
+let audioLogStream = null;
+// Create assistant audio log stream for debugging
+async function createAssistantAudioLogStream(guildId, channelId) {
+    if (!process.env.DEBUG)
+        return null;
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const audioDir = path.join(process.cwd(), 'discord-audio-logs', guildId, channelId);
+    try {
+        await mkdir(audioDir, { recursive: true });
+        // Create stream for assistant audio (24kHz mono s16le PCM)
+        const outputFileName = `assistant_${timestamp}.24.pcm`;
+        const outputFilePath = path.join(audioDir, outputFileName);
+        const outputAudioStream = createWriteStream(outputFilePath);
+        // Add error handler to prevent crashes
+        outputAudioStream.on('error', (error) => {
+            workerLogger.error(`Assistant audio log stream error:`, error);
+        });
+        workerLogger.log(`Created assistant audio log: ${outputFilePath}`);
+        return outputAudioStream;
+    }
+    catch (error) {
+        workerLogger.error(`Failed to create audio log directory:`, error);
+        return null;
+    }
+}
+// Handle encoded Opus packets
+opusEncoder.on('data', (packet) => {
+    opusPacketQueue.push(packet);
+});
+// Handle stream end events
+opusEncoder.on('end', () => {
+    workerLogger.log('Opus encoder stream ended');
+});
+resampler.on('end', () => {
+    workerLogger.log('Resampler stream ended');
+});
+// Handle errors
+resampler.on('error', (error) => {
+    workerLogger.error(`Resampler error:`, error);
+    sendError(`Resampler error: ${error.message}`);
+});
+opusEncoder.on('error', (error) => {
+    workerLogger.error(`Encoder error:`, error);
+    // Check for specific corrupted data errors
+    if (error.message?.includes('The compressed data passed is corrupted')) {
+        workerLogger.warn('Received corrupted audio data in opus encoder');
+    }
+    else {
+        sendError(`Encoder error: ${error.message}`);
+    }
+});
+async function cleanupAsync() {
+    workerLogger.log(`Starting async cleanup`);
+    stopPacketSending();
+    if (session) {
+        workerLogger.log(`Stopping GenAI session`);
+        session.stop();
+        session = null;
+    }
+    // Wait for audio log stream to finish writing
+    if (audioLogStream) {
+        workerLogger.log(`Closing assistant audio log stream`);
+        await new Promise((resolve, reject) => {
+            audioLogStream.end(() => {
+                workerLogger.log(`Assistant audio log stream closed`);
+                resolve();
+            });
+            audioLogStream.on('error', reject);
+            // Add timeout to prevent hanging
+            setTimeout(() => {
+                workerLogger.log(`Audio stream close timeout, continuing`);
+                resolve();
+            }, 3000);
+        });
+        audioLogStream = null;
+    }
+    // Unpipe and end the encoder first
+    resampler.unpipe(opusEncoder);
+    // End the encoder stream
+    await new Promise((resolve) => {
+        opusEncoder.end(() => {
+            workerLogger.log(`Opus encoder ended`);
+            resolve();
+        });
+        // Add timeout
+        setTimeout(resolve, 1000);
+    });
+    // End the resampler stream
+    await new Promise((resolve) => {
+        resampler.end(() => {
+            workerLogger.log(`Resampler ended`);
+            resolve();
+        });
+        // Add timeout
+        setTimeout(resolve, 1000);
+    });
+    workerLogger.log(`Async cleanup complete`);
+}
+// Handle messages from main thread
+parentPort.on('message', async (message) => {
+    try {
+        switch (message.type) {
+            case 'init': {
+                workerLogger.log(`Initializing with directory:`, message.directory);
+                // Create audio log stream for assistant audio
+                audioLogStream = await createAssistantAudioLogStream(message.guildId, message.channelId);
+                // Start packet sending interval
+                startPacketSending();
+                // Get tools for the directory
+                const { tools } = await getTools({
+                    directory: message.directory,
+                    onMessageCompleted: (params) => {
+                        parentPort.postMessage({
+                            type: 'toolCallCompleted',
+                            ...params,
+                        });
+                    },
+                });
+                // Start GenAI session
+                session = await startGenAiSession({
+                    tools,
+                    systemMessage: message.systemMessage,
+                    geminiApiKey: message.geminiApiKey,
+                    onAssistantAudioChunk({ data }) {
+                        // Write to audio log if enabled
+                        if (audioLogStream && !audioLogStream.destroyed) {
+                            audioLogStream.write(data, (err) => {
+                                if (err) {
+                                    workerLogger.error('Error writing to audio log:', err);
+                                }
+                            });
+                        }
+                        // Write PCM data to resampler which will output Opus packets
+                        if (!resampler.destroyed) {
+                            resampler.write(data, (err) => {
+                                if (err) {
+                                    workerLogger.error('Error writing to resampler:', err);
+                                    sendError(`Failed to process audio: ${err.message}`);
+                                }
+                            });
+                        }
+                    },
+                    onAssistantStartSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantStartSpeaking',
+                        });
+                    },
+                    onAssistantStopSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantStopSpeaking',
+                        });
+                    },
+                    onAssistantInterruptSpeaking() {
+                        parentPort.postMessage({
+                            type: 'assistantInterruptSpeaking',
+                        });
+                    },
+                });
+                // Notify main thread we're ready
+                parentPort.postMessage({
+                    type: 'ready',
+                });
+                break;
+            }
+            case 'sendRealtimeInput': {
+                if (!session) {
+                    sendError('Session not initialized');
+                    return;
+                }
+                session.session.sendRealtimeInput({
+                    audio: message.audio,
+                    audioStreamEnd: message.audioStreamEnd,
+                });
+                break;
+            }
+            case 'sendTextInput': {
+                if (!session) {
+                    sendError('Session not initialized');
+                    return;
+                }
+                session.session.sendRealtimeInput({
+                    text: message.text,
+                });
+                break;
+            }
+            case 'interrupt': {
+                workerLogger.log(`Interrupting playback`);
+                // Clear the opus packet queue
+                opusPacketQueue.length = 0;
+                break;
+            }
+            case 'stop': {
+                workerLogger.log(`Stopping worker`);
+                await cleanupAsync();
+                // process.exit(0)
+                break;
+            }
+        }
+    }
+    catch (error) {
+        workerLogger.error(`Error handling message:`, error);
+        sendError(error instanceof Error ? error.message : 'Unknown error in worker');
+    }
+});

package/dist/genai.js

@@ -0,0 +1,232 @@
+// Google GenAI Live session manager for real-time voice interactions.
+// Establishes bidirectional audio streaming with Gemini, handles tool calls,
+// and manages the assistant's audio output for Discord voice channels.
+import { GoogleGenAI, LiveServerMessage, MediaResolution, Modality, Session, } from '@google/genai';
+import { writeFile } from 'fs';
+import { createLogger } from './logger.js';
+import { aiToolToCallableTool } from './ai-tool-to-genai.js';
+const genaiLogger = createLogger('GENAI');
+const audioParts = [];
+function saveBinaryFile(fileName, content) {
+    writeFile(fileName, content, 'utf8', (err) => {
+        if (err) {
+            genaiLogger.error(`Error writing file ${fileName}:`, err);
+            return;
+        }
+        genaiLogger.log(`Appending stream content to file ${fileName}.`);
+    });
+}
+function convertToWav(rawData, mimeType) {
+    const options = parseMimeType(mimeType);
+    const dataLength = rawData.reduce((a, b) => a + b.length, 0);
+    const wavHeader = createWavHeader(dataLength, options);
+    const buffer = Buffer.concat(rawData);
+    return Buffer.concat([wavHeader, buffer]);
+}
+function parseMimeType(mimeType) {
+    const [fileType, ...params] = mimeType.split(';').map((s) => s.trim());
+    const [_, format] = fileType?.split('/') || [];
+    const options = {
+        numChannels: 1,
+        bitsPerSample: 16,
+    };
+    if (format && format.startsWith('L')) {
+        const bits = parseInt(format.slice(1), 10);
+        if (!isNaN(bits)) {
+            options.bitsPerSample = bits;
+        }
+    }
+    for (const param of params) {
+        const [key, value] = param.split('=').map((s) => s.trim());
+        if (key === 'rate') {
+            options.sampleRate = parseInt(value || '', 10);
+        }
+    }
+    return options;
+}
+function createWavHeader(dataLength, options) {
+    const { numChannels, sampleRate, bitsPerSample } = options;
+    // http://soundfile.sapp.org/doc/WaveFormat
+    const byteRate = (sampleRate * numChannels * bitsPerSample) / 8;
+    const blockAlign = (numChannels * bitsPerSample) / 8;
+    const buffer = Buffer.alloc(44);
+    buffer.write('RIFF', 0); // ChunkID
+    buffer.writeUInt32LE(36 + dataLength, 4); // ChunkSize
+    buffer.write('WAVE', 8); // Format
+    buffer.write('fmt ', 12); // Subchunk1ID
+    buffer.writeUInt32LE(16, 16); // Subchunk1Size (PCM)
+    buffer.writeUInt16LE(1, 20); // AudioFormat (1 = PCM)
+    buffer.writeUInt16LE(numChannels, 22); // NumChannels
+    buffer.writeUInt32LE(sampleRate, 24); // SampleRate
+    buffer.writeUInt32LE(byteRate, 28); // ByteRate
+    buffer.writeUInt16LE(blockAlign, 32); // BlockAlign
+    buffer.writeUInt16LE(bitsPerSample, 34); // BitsPerSample
+    buffer.write('data', 36); // Subchunk2ID
+    buffer.writeUInt32LE(dataLength, 40); // Subchunk2Size
+    return buffer;
+}
+function defaultAudioChunkHandler({ data, mimeType, }) {
+    audioParts.push(data);
+    const fileName = 'audio.wav';
+    const buffer = convertToWav(audioParts, mimeType);
+    saveBinaryFile(fileName, buffer);
+}
+export async function startGenAiSession({ onAssistantAudioChunk, onAssistantStartSpeaking, onAssistantStopSpeaking, onAssistantInterruptSpeaking, systemMessage, tools, geminiApiKey, } = {}) {
+    let session = undefined;
+    const callableTools = [];
+    let isAssistantSpeaking = false;
+    const audioChunkHandler = onAssistantAudioChunk || defaultAudioChunkHandler;
+    // Convert AI SDK tools to GenAI CallableTools
+    if (tools) {
+        for (const [name, tool] of Object.entries(tools)) {
+            callableTools.push(aiToolToCallableTool(tool, name));
+        }
+    }
+    function handleModelTurn(message) {
+        if (message.toolCall) {
+            genaiLogger.log('Tool call:', message.toolCall);
+            // Handle tool calls
+            if (message.toolCall.functionCalls && callableTools.length > 0) {
+                for (const tool of callableTools) {
+                    if (!message.toolCall.functionCalls.some((x) => x.name === tool.name)) {
+                        continue;
+                    }
+                    tool
+                        .callTool(message.toolCall.functionCalls)
+                        .then((parts) => {
+                        const functionResponses = parts
+                            .filter((part) => part.functionResponse)
+                            .map((part) => ({
+                            response: part.functionResponse.response,
+                            id: part.functionResponse.id,
+                            name: part.functionResponse.name,
+                        }));
+                        if (functionResponses.length > 0 && session) {
+                            session.sendToolResponse({ functionResponses });
+                            genaiLogger.log('client-toolResponse: ' +
+                                JSON.stringify({ functionResponses }));
+                        }
+                    })
+                        .catch((error) => {
+                        genaiLogger.error('Error handling tool calls:', error);
+                    });
+                }
+            }
+        }
+        if (message.serverContent?.modelTurn?.parts) {
+            for (const part of message.serverContent.modelTurn.parts) {
+                if (part?.fileData) {
+                    genaiLogger.log(`File: ${part?.fileData.fileUri}`);
+                }
+                if (part?.inlineData) {
+                    const inlineData = part.inlineData;
+                    if (!inlineData.mimeType ||
+                        !inlineData.mimeType.startsWith('audio/')) {
+                        genaiLogger.log('Skipping non-audio inlineData:', inlineData.mimeType);
+                        continue;
+                    }
+                    // Trigger start speaking callback the first time audio is received
+                    if (!isAssistantSpeaking && onAssistantStartSpeaking) {
+                        isAssistantSpeaking = true;
+                        onAssistantStartSpeaking();
+                    }
+                    const buffer = Buffer.from(inlineData?.data ?? '', 'base64');
+                    audioChunkHandler({
+                        data: buffer,
+                        mimeType: inlineData.mimeType ?? '',
+                    });
+                }
+                if (part?.text) {
+                    genaiLogger.log('Text:', part.text);
+                }
+            }
+        }
+        // Handle input transcription (user's audio transcription)
+        if (message.serverContent?.inputTranscription?.text) {
+            genaiLogger.log('[user transcription]', message.serverContent.inputTranscription.text);
+        }
+        // Handle output transcription (model's audio transcription)
+        if (message.serverContent?.outputTranscription?.text) {
+            genaiLogger.log('[assistant transcription]', message.serverContent.outputTranscription.text);
+        }
+        if (message.serverContent?.interrupted) {
+            genaiLogger.log('Assistant was interrupted');
+            if (isAssistantSpeaking && onAssistantInterruptSpeaking) {
+                isAssistantSpeaking = false;
+                onAssistantInterruptSpeaking();
+            }
+        }
+        if (message.serverContent?.turnComplete) {
+            genaiLogger.log('Assistant turn complete');
+            if (isAssistantSpeaking && onAssistantStopSpeaking) {
+                isAssistantSpeaking = false;
+                onAssistantStopSpeaking();
+            }
+        }
+    }
+    const apiKey = geminiApiKey || process.env.GEMINI_API_KEY;
+    if (!apiKey) {
+        genaiLogger.error('No Gemini API key provided');
+        throw new Error('Gemini API key is required for voice interactions');
+    }
+    const ai = new GoogleGenAI({
+        apiKey,
+    });
+    const model = 'gemini-2.5-flash-native-audio-preview-12-2025';
+    session = await ai.live.connect({
+        model,
+        callbacks: {
+            onopen: function () {
+                genaiLogger.debug('Opened');
+            },
+            onmessage: function (message) {
+                // genaiLogger.log(message)
+                try {
+                    handleModelTurn(message);
+                }
+                catch (error) {
+                    genaiLogger.error('Error handling turn:', error);
+                }
+            },
+            onerror: function (e) {
+                genaiLogger.debug('Error:', e.message);
+            },
+            onclose: function (e) {
+                genaiLogger.debug('Close:', e.reason);
+            },
+        },
+        config: {
+            tools: callableTools,
+            responseModalities: [Modality.AUDIO],
+            mediaResolution: MediaResolution.MEDIA_RESOLUTION_MEDIUM,
+            inputAudioTranscription: {}, // transcribes your input speech
+            outputAudioTranscription: {}, // transcribes the model's spoken audio
+            systemInstruction: {
+                parts: [
+                    {
+                        text: systemMessage || '',
+                    },
+                ],
+            },
+            speechConfig: {
+                voiceConfig: {
+                    prebuiltVoiceConfig: {
+                        voiceName: 'Charon', // Orus also not bad
+                    },
+                },
+            },
+            contextWindowCompression: {
+                triggerTokens: '25600',
+                slidingWindow: { targetTokens: '12800' },
+            },
+        },
+    });
+    return {
+        session,
+        stop: () => {
+            const currentSession = session;
+            session = undefined;
+            currentSession?.close();
+        },
+    };
+}

package/dist/interaction-handler.js

@@ -0,0 +1,144 @@
+// Discord slash command and interaction handler.
+// Processes all slash commands (/session, /resume, /fork, /model, /abort, etc.)
+// and manages autocomplete, select menu interactions for the bot.
+import { Events } from 'discord.js';
+import { handleSessionCommand, handleSessionAutocomplete } from './commands/session.js';
+import { handleResumeCommand, handleResumeAutocomplete } from './commands/resume.js';
+import { handleAddProjectCommand, handleAddProjectAutocomplete } from './commands/add-project.js';
+import { handleCreateNewProjectCommand } from './commands/create-new-project.js';
+import { handleAcceptCommand, handleRejectCommand } from './commands/permissions.js';
+import { handleAbortCommand } from './commands/abort.js';
+import { handleShareCommand } from './commands/share.js';
+import { handleForkCommand, handleForkSelectMenu } from './commands/fork.js';
+import { handleModelCommand, handleProviderSelectMenu, handleModelSelectMenu } from './commands/model.js';
+import { handleAgentCommand, handleAgentSelectMenu } from './commands/agent.js';
+import { handleAskQuestionSelectMenu } from './commands/ask-question.js';
+import { handleQueueCommand, handleClearQueueCommand } from './commands/queue.js';
+import { handleUndoCommand, handleRedoCommand } from './commands/undo-redo.js';
+import { handleUserCommand } from './commands/user-command.js';
+import { createLogger } from './logger.js';
+const interactionLogger = createLogger('INTERACTION');
+export function registerInteractionHandler({ discordClient, appId, }) {
+    interactionLogger.log('[REGISTER] Interaction handler registered');
+    discordClient.on(Events.InteractionCreate, async (interaction) => {
+        try {
+            interactionLogger.log(`[INTERACTION] Received: ${interaction.type} - ${interaction.isChatInputCommand()
+                ? interaction.commandName
+                : interaction.isAutocomplete()
+                    ? `autocomplete:${interaction.commandName}`
+                    : 'other'}`);
+            if (interaction.isAutocomplete()) {
+                switch (interaction.commandName) {
+                    case 'session':
+                        await handleSessionAutocomplete({ interaction, appId });
+                        return;
+                    case 'resume':
+                        await handleResumeAutocomplete({ interaction, appId });
+                        return;
+                    case 'add-project':
+                        await handleAddProjectAutocomplete({ interaction, appId });
+                        return;
+                    default:
+                        await interaction.respond([]);
+                        return;
+                }
+            }
+            if (interaction.isChatInputCommand()) {
+                interactionLogger.log(`[COMMAND] Processing: ${interaction.commandName}`);
+                switch (interaction.commandName) {
+                    case 'session':
+                        await handleSessionCommand({ command: interaction, appId });
+                        return;
+                    case 'resume':
+                        await handleResumeCommand({ command: interaction, appId });
+                        return;
+                    case 'add-project':
+                        await handleAddProjectCommand({ command: interaction, appId });
+                        return;
+                    case 'create-new-project':
+                        await handleCreateNewProjectCommand({ command: interaction, appId });
+                        return;
+                    case 'accept':
+                    case 'accept-always':
+                        await handleAcceptCommand({ command: interaction, appId });
+                        return;
+                    case 'reject':
+                        await handleRejectCommand({ command: interaction, appId });
+                        return;
+                    case 'abort':
+                    case 'stop':
+                        await handleAbortCommand({ command: interaction, appId });
+                        return;
+                    case 'share':
+                        await handleShareCommand({ command: interaction, appId });
+                        return;
+                    case 'fork':
+                        await handleForkCommand(interaction);
+                        return;
+                    case 'model':
+                        await handleModelCommand({ interaction, appId });
+                        return;
+                    case 'agent':
+                        await handleAgentCommand({ interaction, appId });
+                        return;
+                    case 'queue':
+                        await handleQueueCommand({ command: interaction, appId });
+                        return;
+                    case 'clear-queue':
+                        await handleClearQueueCommand({ command: interaction, appId });
+                        return;
+                    case 'undo':
+                        await handleUndoCommand({ command: interaction, appId });
+                        return;
+                    case 'redo':
+                        await handleRedoCommand({ command: interaction, appId });
+                        return;
+                }
+                // Handle user-defined commands (ending with -cmd suffix)
+                if (interaction.commandName.endsWith('-cmd')) {
+                    await handleUserCommand({ command: interaction, appId });
+                    return;
+                }
+                return;
+            }
+            if (interaction.isStringSelectMenu()) {
+                const customId = interaction.customId;
+                if (customId.startsWith('fork_select:')) {
+                    await handleForkSelectMenu(interaction);
+                    return;
+                }
+                if (customId.startsWith('model_provider:')) {
+                    await handleProviderSelectMenu(interaction);
+                    return;
+                }
+                if (customId.startsWith('model_select:')) {
+                    await handleModelSelectMenu(interaction);
+                    return;
+                }
+                if (customId.startsWith('agent_select:')) {
+                    await handleAgentSelectMenu(interaction);
+                    return;
+                }
+                if (customId.startsWith('ask_question:')) {
+                    await handleAskQuestionSelectMenu(interaction);
+                    return;
+                }
+                return;
+            }
+        }
+        catch (error) {
+            interactionLogger.error('[INTERACTION] Error handling interaction:', error);
+            try {
+                if (interaction.isRepliable() && !interaction.replied && !interaction.deferred) {
+                    await interaction.reply({
+                        content: 'An error occurred processing this command.',
+                        ephemeral: true,
+                    });
+                }
+            }
+            catch (replyError) {
+                interactionLogger.error('[INTERACTION] Failed to send error reply:', replyError);
+            }
+        }
+    });
+}