shuvmaki 0.4.26

package/src/voice.ts

@@ -0,0 +1,432 @@
+// Audio transcription service using Google Gemini.
+// Transcribes voice messages with code-aware context, using grep/glob tools
+// to verify technical terms, filenames, and function names in the codebase.
+
+import {
+  GoogleGenAI,
+  Type,
+  type Content,
+  type Part,
+  type Tool,
+} from '@google/genai'
+import { createLogger } from './logger.js'
+import { glob } from 'glob'
+import { ripGrep } from 'ripgrep-js'
+
+const voiceLogger = createLogger('VOICE')
+
+export type TranscriptionToolRunner = ({
+  name,
+  args,
+}: {
+  name: string
+  args: Record<string, string> | undefined
+}) => Promise<
+  | { type: 'result'; transcription: string }
+  | { type: 'toolResponse'; name: string; output: string }
+  | { type: 'skip' }
+>
+
+async function runGrep({
+  pattern,
+  directory,
+}: {
+  pattern: string
+  directory: string
+}): Promise<string> {
+  try {
+    const results = await ripGrep(directory, {
+      string: pattern,
+      globs: ['!node_modules/**', '!.git/**', '!dist/**', '!build/**'],
+    })
+
+    if (results.length === 0) {
+      return 'No matches found'
+    }
+
+    const output = results
+      .slice(0, 10)
+      .map((match) => {
+        return `${match.path.text}:${match.line_number}: ${match.lines.text.trim()}`
+      })
+      .join('\n')
+
+    return output.slice(0, 2000)
+  } catch (e) {
+    voiceLogger.error('grep search failed:', e)
+    return 'grep search failed'
+  }
+}
+
+async function runGlob({
+  pattern,
+  directory,
+}: {
+  pattern: string
+  directory: string
+}): Promise<string> {
+  try {
+    const files = await glob(pattern, {
+      cwd: directory,
+      nodir: false,
+      ignore: ['node_modules/**', '.git/**', 'dist/**', 'build/**'],
+      maxDepth: 10,
+    })
+
+    if (files.length === 0) {
+      return 'No files found'
+    }
+
+    return files.slice(0, 30).join('\n')
+  } catch (error) {
+    return `Glob search failed: ${error instanceof Error ? error.message : 'Unknown error'}`
+  }
+}
+
+const grepToolDeclaration = {
+  name: 'grep',
+  description:
+    'Search for a pattern in file contents to verify if a technical term, function name, or variable exists in the code. Use this to check if transcribed words match actual code.',
+  parameters: {
+    type: Type.OBJECT,
+    properties: {
+      pattern: {
+        type: Type.STRING,
+        description:
+          'The search pattern (case-insensitive). Can be a word, function name, or partial match.',
+      },
+    },
+    required: ['pattern'],
+  },
+}
+
+const globToolDeclaration = {
+  name: 'glob',
+  description:
+    'Search for files by name pattern. Use this to verify if a filename or directory mentioned in the audio actually exists in the project.',
+  parameters: {
+    type: Type.OBJECT,
+    properties: {
+      pattern: {
+        type: Type.STRING,
+        description:
+          'The glob pattern to match files. Examples: "*.ts", "**/*.json", "**/config*", "src/**/*.tsx"',
+      },
+    },
+    required: ['pattern'],
+  },
+}
+
+const transcriptionResultToolDeclaration = {
+  name: 'transcriptionResult',
+  description:
+    'MANDATORY: You MUST call this tool to complete the task. This is the ONLY way to return results - text responses are ignored. Call this with your transcription, even if imperfect. An imperfect transcription is better than none.',
+  parameters: {
+    type: Type.OBJECT,
+    properties: {
+      transcription: {
+        type: Type.STRING,
+        description:
+          'The final transcription of the audio. MUST be non-empty. If audio is unclear, transcribe your best interpretation. If silent, use "[inaudible audio]".',
+      },
+    },
+    required: ['transcription'],
+  },
+}
+
+function createToolRunner({
+  directory,
+}: {
+  directory?: string
+}): TranscriptionToolRunner {
+  const hasDirectory = directory && directory.trim().length > 0
+
+  return async ({ name, args }) => {
+    if (name === 'transcriptionResult') {
+      return {
+        type: 'result',
+        transcription: args?.transcription || '',
+      }
+    }
+
+    if (name === 'grep' && hasDirectory) {
+      const pattern = args?.pattern || ''
+      voiceLogger.log(`Grep search: "${pattern}"`)
+      const output = await runGrep({ pattern, directory })
+      voiceLogger.log(`Grep result: ${output.slice(0, 100)}...`)
+      return { type: 'toolResponse', name: 'grep', output }
+    }
+
+    if (name === 'glob' && hasDirectory) {
+      const pattern = args?.pattern || ''
+      voiceLogger.log(`Glob search: "${pattern}"`)
+      const output = await runGlob({ pattern, directory })
+      voiceLogger.log(`Glob result: ${output.slice(0, 100)}...`)
+      return { type: 'toolResponse', name: 'glob', output }
+    }
+
+    return { type: 'skip' }
+  }
+}
+
+export async function runTranscriptionLoop({
+  genAI,
+  model,
+  initialContents,
+  tools,
+  temperature,
+  toolRunner,
+  maxSteps = 10,
+}: {
+  genAI: GoogleGenAI
+  model: string
+  initialContents: Content[]
+  tools: Tool[]
+  temperature: number
+  toolRunner: TranscriptionToolRunner
+  maxSteps?: number
+}): Promise<string> {
+  let response = await genAI.models.generateContent({
+    model,
+    contents: initialContents,
+    config: {
+      temperature,
+      thinkingConfig: {
+        thinkingBudget: 1024,
+      },
+      tools,
+    },
+  })
+
+  const conversationHistory: Content[] = [...initialContents]
+  let stepsRemaining = maxSteps
+
+  while (true) {
+    const candidate = response.candidates?.[0]
+    if (!candidate?.content?.parts) {
+      const text = response.text?.trim()
+      if (text) {
+        voiceLogger.log(`No parts but got text response: "${text.slice(0, 100)}..."`)
+        return text
+      }
+      throw new Error('Transcription failed: No response content from model')
+    }
+
+    const functionCalls = candidate.content.parts.filter(
+      (part): part is Part & { functionCall: NonNullable<Part['functionCall']> } =>
+        'functionCall' in part && !!part.functionCall,
+    )
+
+    if (functionCalls.length === 0) {
+      const text = response.text?.trim()
+      if (text) {
+        voiceLogger.log(`No function calls but got text: "${text.slice(0, 100)}..."`)
+        return text
+      }
+      throw new Error('Transcription failed: Model did not produce a transcription')
+    }
+
+    conversationHistory.push({
+      role: 'model',
+      parts: candidate.content.parts,
+    })
+
+    const functionResponseParts: Array<{
+      functionResponse: { name: string; response: { output: string } }
+    }> = []
+
+    for (const part of functionCalls) {
+      const call = part.functionCall
+      const args = call.args as Record<string, string> | undefined
+      const result = await toolRunner({ name: call.name || '', args })
+
+      if (result.type === 'result') {
+        const transcription = result.transcription?.trim() || ''
+        voiceLogger.log(
+          `Transcription result received: "${transcription.slice(0, 100)}..."`,
+        )
+        if (!transcription) {
+          throw new Error('Transcription failed: Model returned empty transcription')
+        }
+        return transcription
+      }
+
+      if (result.type === 'toolResponse') {
+        stepsRemaining--
+        const stepsWarning: string = (() => {
+          if (stepsRemaining <= 0) {
+            return '\n\n[CRITICAL: Tool limit reached. You MUST call transcriptionResult NOW. No more grep/glob allowed. Call transcriptionResult immediately with your best transcription.]'
+          }
+          if (stepsRemaining === 1) {
+            return '\n\n[URGENT: FINAL STEP. You MUST call transcriptionResult NOW. Do NOT call grep or glob. Call transcriptionResult with your transcription immediately.]'
+          }
+          if (stepsRemaining <= 3) {
+            return `\n\n[WARNING: Only ${stepsRemaining} steps remaining. Finish searching soon and call transcriptionResult. Do not wait until the last step.]`
+          }
+          return ''
+        })()
+
+        functionResponseParts.push({
+          functionResponse: {
+            name: result.name,
+            response: { output: result.output + stepsWarning },
+          },
+        })
+      }
+    }
+
+    if (functionResponseParts.length === 0) {
+      throw new Error('Transcription failed: No valid tool responses')
+    }
+
+    conversationHistory.push({
+      role: 'user',
+      parts: functionResponseParts,
+    } as Content)
+
+    response = await genAI.models.generateContent({
+      model,
+      contents: conversationHistory,
+      config: {
+        temperature,
+        thinkingConfig: {
+          thinkingBudget: 512,
+        },
+        tools: stepsRemaining <= 0 ? [{ functionDeclarations: [transcriptionResultToolDeclaration] }] : tools,
+      },
+    })
+  }
+}
+
+export async function transcribeAudio({
+  audio,
+  prompt,
+  language,
+  temperature,
+  geminiApiKey,
+  directory,
+  currentSessionContext,
+  lastSessionContext,
+}: {
+  audio: Buffer | Uint8Array | ArrayBuffer | string
+  prompt?: string
+  language?: string
+  temperature?: number
+  geminiApiKey?: string
+  directory?: string
+  currentSessionContext?: string
+  lastSessionContext?: string
+}): Promise<string> {
+  try {
+    const apiKey = geminiApiKey || process.env.GEMINI_API_KEY
+
+    if (!apiKey) {
+      throw new Error('Gemini API key is required for audio transcription')
+    }
+
+    const genAI = new GoogleGenAI({ apiKey })
+
+    let audioBase64: string
+    if (typeof audio === 'string') {
+      audioBase64 = audio
+    } else if (audio instanceof Buffer) {
+      audioBase64 = audio.toString('base64')
+    } else if (audio instanceof Uint8Array) {
+      audioBase64 = Buffer.from(audio).toString('base64')
+    } else if (audio instanceof ArrayBuffer) {
+      audioBase64 = Buffer.from(audio).toString('base64')
+    } else {
+      throw new Error('Invalid audio format')
+    }
+
+    const languageHint = language ? `The audio is in ${language}.\n\n` : ''
+
+    // build session context section
+    const sessionContextParts: string[] = []
+    if (lastSessionContext) {
+      sessionContextParts.push(`<last_session>
+${lastSessionContext}
+</last_session>`)
+    }
+    if (currentSessionContext) {
+      sessionContextParts.push(`<current_session>
+${currentSessionContext}
+</current_session>`)
+    }
+    const sessionContextSection = sessionContextParts.length > 0
+      ? `\nSession context (use to understand references to files, functions, tools used):\n${sessionContextParts.join('\n\n')}`
+      : ''
+
+    const transcriptionPrompt = `${languageHint}Transcribe this audio for a coding agent (like Claude Code or OpenCode).
+
+CRITICAL REQUIREMENT: You MUST call the "transcriptionResult" tool to complete this task.
+- The transcriptionResult tool is the ONLY way to return results
+- Text responses are completely ignored - only tool calls work
+- You MUST call transcriptionResult even if you run out of tool calls
+- An imperfect transcription is better than no transcription
+- DO NOT end without calling transcriptionResult
+
+This is a software development environment. The speaker is giving instructions to an AI coding assistant. Expect:
+- File paths, function names, CLI commands, package names, API endpoints
+
+RULES:
+1. If audio is unclear, transcribe your best interpretation, interpreting words event with strong accents are present, identifying the accent being used first so you can guess what the words meawn
+2. If audio seems silent/empty, call transcriptionResult with "[inaudible audio]"
+3. Use the session context below to understand technical terms, file names, function names mentioned
+
+Common corrections (apply without tool calls):
+- "reacked" → "React", "jason" → "JSON", "get hub" → "GitHub", "no JS" → "Node.js", "dacker" → "Docker"
+
+Project file structure:
+<file_tree>
+${prompt}
+</file_tree>
+${sessionContextSection}
+
+REMEMBER: Call "transcriptionResult" tool with your transcription. This is mandatory.
+
+Note: "critique" is a CLI tool for showing diffs in the browser.`
+
+    // const hasDirectory = directory && directory.trim().length > 0
+    const tools = [
+      {
+        functionDeclarations: [
+          transcriptionResultToolDeclaration,
+          // grep/glob disabled - was causing transcription to hang
+          // ...(hasDirectory ? [grepToolDeclaration, globToolDeclaration] : []),
+        ],
+      },
+    ]
+
+    const initialContents: Content[] = [
+      {
+        role: 'user',
+        parts: [
+          { text: transcriptionPrompt },
+          {
+            inlineData: {
+              data: audioBase64,
+              mimeType: 'audio/mpeg',
+            },
+          },
+        ],
+      },
+    ]
+
+    const toolRunner = createToolRunner({ directory })
+
+    return await runTranscriptionLoop({
+      genAI,
+      model: 'gemini-2.5-flash',
+      initialContents,
+      tools,
+      temperature: temperature ?? 0.3,
+      toolRunner,
+    })
+  } catch (error) {
+    voiceLogger.error('Failed to transcribe audio:', error)
+    throw new Error(
+      `Audio transcription failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
+    )
+  }
+}

package/src/worker-types.ts

@@ -0,0 +1,66 @@
+// Type definitions for worker thread message passing.
+// Defines the protocol between main thread and GenAI worker for
+// audio streaming, tool calls, and session lifecycle management.
+
+import type { Tool as AITool } from 'ai'
+
+// Messages sent from main thread to worker
+export type WorkerInMessage =
+  | {
+      type: 'init'
+      directory: string // Project directory for tools
+      systemMessage?: string
+      guildId: string
+      channelId: string
+      appId: string
+      geminiApiKey?: string | null
+    }
+  | {
+      type: 'sendRealtimeInput'
+      audio?: {
+        mimeType: string
+        data: string // base64
+      }
+      audioStreamEnd?: boolean
+    }
+  | {
+      type: 'sendTextInput'
+      text: string
+    }
+  | {
+      type: 'interrupt'
+    }
+  | {
+      type: 'stop'
+    }
+
+// Messages sent from worker to main thread via parentPort
+export type WorkerOutMessage =
+  | {
+      type: 'assistantOpusPacket'
+      packet: ArrayBuffer // Opus encoded audio packet
+    }
+  | {
+      type: 'assistantStartSpeaking'
+    }
+  | {
+      type: 'assistantStopSpeaking'
+    }
+  | {
+      type: 'assistantInterruptSpeaking'
+    }
+  | {
+      type: 'toolCallCompleted'
+      sessionId: string
+      messageId: string
+      data?: any
+      error?: any
+      markdown?: string
+    }
+  | {
+      type: 'error'
+      error: string
+    }
+  | {
+      type: 'ready'
+    }

package/src/xml.test.ts

@@ -0,0 +1,37 @@
+import { describe, test, expect } from 'vitest'
+import { extractNonXmlContent } from './xml.js'
+
+describe('extractNonXmlContent', () => {
+  test('removes xml tags and returns only text content', () => {
+    const xml = 'Hello <tag>content</tag> world <nested><inner>deep</inner></nested> end'
+    expect(extractNonXmlContent(xml)).toMatchInlineSnapshot(`
+      "Hello
+      world
+      end"
+    `)
+  })
+
+  test('handles multiple text segments', () => {
+    const xml = 'Start <a>tag1</a> middle <b>tag2</b> finish'
+    expect(extractNonXmlContent(xml)).toMatchInlineSnapshot(`
+      "Start
+      middle
+      finish"
+    `)
+  })
+
+  test('handles only xml without text', () => {
+    const xml = '<root><child>content</child></root>'
+    expect(extractNonXmlContent(xml)).toMatchInlineSnapshot(`""`)
+  })
+
+  test('handles only text without xml', () => {
+    const xml = 'Just plain text'
+    expect(extractNonXmlContent(xml)).toMatchInlineSnapshot(`"Just plain text"`)
+  })
+
+  test('handles empty string', () => {
+    const xml = ''
+    expect(extractNonXmlContent(xml)).toMatchInlineSnapshot(`""`)
+  })
+})

package/src/xml.ts

@@ -0,0 +1,121 @@
+// XML/HTML tag content extractor.
+// Parses XML-like tags from strings (e.g., channel topics) to extract
+// Kimaki configuration like directory paths and app IDs.
+
+import { DomHandler, Parser, ElementType } from 'htmlparser2'
+import type { ChildNode, Element, Text } from 'domhandler'
+import { createLogger } from './logger.js'
+
+const xmlLogger = createLogger('XML')
+
+export function extractTagsArrays<T extends string>({
+  xml,
+  tags,
+}: {
+  xml: string
+  tags: T[]
+}): Record<T, string[]> & { others: string[] } {
+  const result: Record<string, string[]> = {
+    others: [],
+  }
+
+  // Initialize arrays for each tag
+  tags.forEach((tag) => {
+    result[tag] = []
+  })
+
+  try {
+    const handler = new DomHandler(
+      (error, dom) => {
+        if (error) {
+          xmlLogger.error('Error parsing XML:', error)
+        } else {
+          const findTags = (nodes: ChildNode[], path: string[] = []) => {
+            nodes.forEach((node) => {
+              if (node.type === ElementType.Tag) {
+                const element = node as Element
+                const currentPath = [...path, element.name]
+                const pathString = currentPath.join('.')
+
+                // Extract content using original string positions
+                const extractContent = (): string => {
+                  // Use element's own indices but exclude the tags
+                  if (
+                    element.startIndex !== null &&
+                    element.endIndex !== null
+                  ) {
+                    // Extract the full element including tags
+                    const fullElement = xml.substring(
+                      element.startIndex,
+                      element.endIndex + 1,
+                    )
+                    // Find where content starts (after opening tag)
+                    const contentStart = fullElement.indexOf('>') + 1
+                    // Find where content ends (before this element's closing tag)
+                    const closingTag = `</${element.name}>`
+                    const contentEnd = fullElement.lastIndexOf(closingTag)
+
+                    if (contentStart > 0 && contentEnd > contentStart) {
+                      return fullElement.substring(contentStart, contentEnd)
+                    }
+
+                    return ''
+                  }
+                  return ''
+                }
+
+                // Check both single tag names and nested paths
+                if (tags.includes(element.name as T)) {
+                  const content = extractContent()
+                  result[element.name as T]?.push(content)
+                }
+
+                // Check for nested path matches
+                if (tags.includes(pathString as T)) {
+                  const content = extractContent()
+                  result[pathString as T]?.push(content)
+                }
+
+                if (element.children) {
+                  findTags(element.children, currentPath)
+                }
+              } else if (
+                node.type === ElementType.Text &&
+                node.parent?.type === ElementType.Root
+              ) {
+                const textNode = node as Text
+                if (textNode.data.trim()) {
+                  // console.log('node.parent',node.parent)
+                  result.others?.push(textNode.data.trim())
+                }
+              }
+            })
+          }
+
+          findTags(dom)
+        }
+      },
+      {
+        withStartIndices: true,
+        withEndIndices: true,
+        xmlMode: true,
+      },
+    )
+
+    const parser = new Parser(handler, {
+      xmlMode: true,
+      decodeEntities: false,
+    })
+    parser.write(xml)
+    parser.end()
+  } catch (error) {
+    xmlLogger.error('Unexpected error in extractTags:', error)
+  }
+
+  return result as Record<T, string[]> & { others: string[] }
+}
+
+export function extractNonXmlContent(xml: string): string {
+  const result = extractTagsArrays({ xml, tags: [] })
+  return result.others.join('\n')
+}