zerg-ztc 0.1.7 → 0.1.11

package/src/agent/agent.ts

@@ -4,7 +4,7 @@ import { extname } from 'path';
 
 // Local
 import { Message, ToolCall, AgentEvent } from '../types.js';
-import { AnthropicBackend, AgentBackend, BackendRequest, BackendResponse, ContentBlock, LlmMessage, RequestContentBlock, TokenUsage } from './backends/index.js';
+import { AnthropicBackend, AgentBackend, BackendRequest, BackendResponse, ContentBlock, LlmMessage, RequestContentBlock, ToolResultBlock, TokenUsage } from './backends/index.js';
 import { AllowAllPolicy, Policy } from './runtime/policy.js';
 import { NoopTracer, Tracer } from './runtime/tracing.js';
 import { defaultTools, executeTool, getToolDefinitions, getTool } from './tools/index.js';
@@ -85,8 +85,11 @@ export class Agent {
 
 You have access to tools for:
 - Reading and writing files
-- Listing directory contents  
+- Listing directory contents
 - Running shell commands
+- Taking screenshots (full screen or specific windows by app name, PID, or window ID)
+- Listing open windows to find window IDs
+- Launching apps and capturing their windows
 - Querying the Zerg system
 
 Be concise and helpful. When using tools, explain what you're doing briefly. If a task requires multiple steps, proceed through them systematically.
@@ -112,29 +115,131 @@ When a user intent maps to an available slash command, invoke the command direct
       .filter((m): m is Message & { role: 'user' | 'assistant' } => {
         // Only include user and assistant messages
         if (m.role !== 'user' && m.role !== 'assistant') return false;
-        // Filter out assistant messages with empty content (from tool-only responses)
-        // The API rejects empty content for non-final assistant messages
-        if (m.role === 'assistant' && (!m.content || m.content.trim() === '')) return false;
+        // Filter out assistant messages with empty content AND no tool calls
+        if (m.role === 'assistant' && (!m.content || m.content.trim() === '') && !m.toolCalls?.length) return false;
         return true;
       })
-      .map(m => ({
-        role: m.role,
-        content: m.role === 'user' ? this.buildContentBlocks(m.content) : m.content
-      }));
+      .map(m => {
+        if (m.role === 'user') {
+          return {
+            role: m.role,
+            content: this.buildContentBlocks(m.content)
+          };
+        }
+
+        // Assistant message - may need to include tool_use blocks
+        if (m.toolCalls && m.toolCalls.length > 0) {
+          // Build content array with text and tool_use blocks
+          const contentBlocks: Array<{ type: 'text'; text: string } | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }> = [];
+
+          // Add text content if present
+          if (m.content && m.content.trim() && m.content !== '[Using tools...]') {
+            contentBlocks.push({ type: 'text', text: m.content });
+          }
+
+          // Add tool_use blocks
+          for (const tc of m.toolCalls) {
+            contentBlocks.push({
+              type: 'tool_use',
+              id: tc.id,
+              name: tc.name,
+              input: tc.args
+            });
+          }
+
+          return {
+            role: m.role,
+            content: contentBlocks as unknown as RequestContentBlock[]
+          };
+        }
+
+        // Plain text assistant message
+        return {
+          role: m.role,
+          content: m.content
+        };
+      });
   }
 
   private contentLength(content: string | RequestContentBlock[]): number {
     if (typeof content === 'string') return content.length;
     return content.reduce((sum, block) => {
       if (block.type === 'text') return sum + block.text.length;
-      return sum + block.data.length;
+      if (block.type === 'image') return sum + block.data.length;
+      if (block.type === 'tool_result') {
+        // Estimate tool result content length
+        if (typeof block.content === 'string') return sum + block.content.length;
+        return sum + block.content.reduce((s, b) => {
+          if (b.type === 'text') return s + b.text.length;
+          if (b.type === 'image') return s + b.source.data.length;
+          return s;
+        }, 0);
+      }
+      return sum;
     }, 0);
   }
 
+  // Parse tool result string to check for image data
+  private parseToolResultForImages(result: string): { hasImage: boolean; imageData?: { mediaType: string; data: string }; text: string } {
+    try {
+      const parsed = JSON.parse(result);
+      if (parsed && parsed.type === 'image' && parsed.data && parsed.mediaType) {
+        return {
+          hasImage: true,
+          imageData: { mediaType: parsed.mediaType, data: parsed.data },
+          text: parsed.description || 'Screenshot captured'
+        };
+      }
+    } catch {
+      // Not JSON or not an image result
+    }
+    return { hasImage: false, text: result };
+  }
+
+  // Build tool result content blocks for the API
+  private buildToolResultBlocks(toolResults: Array<{ tool_use_id: string; content: string }>): ToolResultBlock[] {
+    return toolResults.map(result => {
+      const parsed = this.parseToolResultForImages(result.content);
+
+      if (parsed.hasImage && parsed.imageData) {
+        // Include both text and image in tool result
+        return {
+          type: 'tool_result' as const,
+          tool_use_id: result.tool_use_id,
+          content: [
+            { type: 'text' as const, text: parsed.text },
+            {
+              type: 'image' as const,
+              source: {
+                type: 'base64' as const,
+                media_type: parsed.imageData.mediaType,
+                data: parsed.imageData.data
+              }
+            }
+          ]
+        };
+      }
+
+      // Plain text result
+      return {
+        type: 'tool_result' as const,
+        tool_use_id: result.tool_use_id,
+        content: result.content
+      };
+    });
+  }
+
   private buildContentBlocks(content: string): string | RequestContentBlock[] {
     const trimmed = content.trimStart();
+
+    // Check if this is a tool results message
     if (trimmed.startsWith('[') && trimmed.includes('"tool_use_id"')) {
-      return content;
+      try {
+        const toolResults = JSON.parse(trimmed) as Array<{ tool_use_id: string; content: string }>;
+        return this.buildToolResultBlocks(toolResults);
+      } catch {
+        return content;
+      }
     }
 
     const imageRegex = /\[image ([^\]]+)\]/g;

package/src/agent/backends/anthropic.ts

@@ -32,11 +32,21 @@ export class AnthropicBackend implements AgentBackend {
         role: message.role,
         content: typeof message.content === 'string'
           ? message.content
-          : message.content.map(block => (
-            block.type === 'text'
-              ? { type: 'text', text: block.text }
-              : { type: 'image', source: { type: 'base64', media_type: block.mediaType, data: block.data } }
-          ))
+          : message.content.map(block => {
+            if (block.type === 'text') {
+              return { type: 'text', text: block.text };
+            }
+            if (block.type === 'tool_result') {
+              // Pass tool results through in Anthropic format
+              return block;
+            }
+            if (block.type === 'tool_use') {
+              // Pass tool_use blocks through for assistant messages
+              return block;
+            }
+            // Image block
+            return { type: 'image', source: { type: 'base64', media_type: block.mediaType, data: block.data } };
+          })
       })),
       tools: request.tools.map(t => ({
         name: t.name,

package/src/agent/backends/gemini.ts

@@ -46,6 +46,18 @@ export class GeminiBackend implements AgentBackend {
         if (block.type === 'text') {
           return { text: block.text };
         }
+        if (block.type === 'tool_result') {
+          // Gemini handles function responses differently - convert to text for now
+          const resultText = typeof block.content === 'string'
+            ? block.content
+            : block.content.map(b => b.type === 'text' ? b.text : '[image]').join('\n');
+          return { text: `Function result: ${resultText}` };
+        }
+        if (block.type === 'tool_use') {
+          // Convert tool_use to function call format for Gemini
+          return { functionCall: { name: block.name, args: block.input } };
+        }
+        // Image block
         return { inlineData: { mimeType: block.mediaType, data: block.data } };
       });
     };

package/src/agent/backends/index.ts

@@ -12,5 +12,6 @@ export type {
   ToolUseBlock,
   LlmMessage,
   RequestContentBlock,
+  ToolResultBlock,
   TokenUsage
 } from './types.js';

package/src/agent/backends/openai_compatible.ts

@@ -49,6 +49,18 @@ export class OpenAICompatibleBackend implements AgentBackend {
         if (block.type === 'text') {
           return { type: 'text', text: block.text };
         }
+        if (block.type === 'tool_result') {
+          // OpenAI format: convert tool result to text
+          const resultText = typeof block.content === 'string'
+            ? block.content
+            : block.content.map(b => b.type === 'text' ? b.text : '[image]').join('\n');
+          return { type: 'text', text: `Tool result (${block.tool_use_id}): ${resultText}` };
+        }
+        if (block.type === 'tool_use') {
+          // OpenAI handles tool calls differently - convert to text representation
+          return { type: 'text', text: `[Tool call: ${block.name}(${JSON.stringify(block.input)})]` };
+        }
+        // Image block
         return {
           type: 'image_url',
           image_url: { url: `data:${block.mediaType};base64,${block.data}` }

package/src/agent/backends/types.ts

@@ -12,7 +12,31 @@ export interface RequestImageBlock {
   path?: string;
 }
 
-export type RequestContentBlock = RequestTextBlock | RequestImageBlock;
+export interface ToolResultImageSource {
+  type: 'base64';
+  media_type: string;
+  data: string;
+}
+
+export interface ToolResultImageBlock {
+  type: 'image';
+  source: ToolResultImageSource;
+}
+
+export interface ToolResultBlock {
+  type: 'tool_result';
+  tool_use_id: string;
+  content: string | Array<RequestTextBlock | ToolResultImageBlock>;
+}
+
+export interface ToolUseRequestBlock {
+  type: 'tool_use';
+  id: string;
+  name: string;
+  input: Record<string, unknown>;
+}
+
+export type RequestContentBlock = RequestTextBlock | RequestImageBlock | ToolResultBlock | ToolUseRequestBlock;
 
 export interface LlmMessage {
   role: 'user' | 'assistant';

package/src/agent/commands/dictation.ts

@@ -0,0 +1,11 @@
+import { Command } from './types.js';
+import { getDictationStatus } from '../../utils/dictation.js';
+
+export const dictationStatusCommand: Command = {
+  name: 'dictation',
+  description: 'Check voice dictation status and availability',
+  handler: async (args, ctx) => {
+    const status = getDictationStatus();
+    ctx.addMessage({ role: 'system', content: status });
+  }
+};

package/src/agent/commands/index.ts

@@ -15,6 +15,7 @@ import { retryCommand } from './retry.js';
 import { inputModeCommand } from './input_mode.js';
 import { keybindingsCommand } from './keybindings.js';
 import { updateCommand } from './update.js';
+import { dictationStatusCommand } from './dictation.js';
 import { Command } from './types.js';
 
 const commandList: Command[] = [];
@@ -39,6 +40,7 @@ commandList.push(
   updateCommand,
   inputModeCommand,
   retryCommand,
+  dictationStatusCommand,
   exitCommand
 );
 

package/src/agent/commands/types.ts

@@ -53,6 +53,12 @@ export interface SkillsController {
   list: () => Promise<Skill[]>;
 }
 
+export interface DictationController {
+  startRecording: () => void;
+  stopRecording: () => Promise<string>;  // Returns transcribed text
+  isRecording: () => boolean;
+}
+
 export interface CommandContext {
   addMessage: (msg: Omit<Message, 'id' | 'timestamp'>) => void;
   clearMessages: () => void;
@@ -68,8 +74,10 @@ export interface CommandContext {
   clipboard: ClipboardController;
   models: ModelsController;
   skills: SkillsController;
+  dictation?: DictationController;
   getInputMode: () => 'queue' | 'interrupt';
   setInputMode: (mode: 'queue' | 'interrupt') => void;
+  setInputText?: (text: string) => void;  // Set input field text
 }
 
 export interface Command {

package/src/agent/runtime/capabilities.ts

@@ -2,5 +2,6 @@ export enum ToolCapability {
   FILE_READ = 'file_read',
   FILE_WRITE = 'file_write',
   SHELL_EXEC = 'shell_exec',
-  NETWORK = 'network'
+  NETWORK = 'network',
+  SCREEN_CAPTURE = 'screen_capture'
 }

package/src/agent/tools/index.ts

@@ -5,6 +5,7 @@ import { runCommandTool } from './shell.js';
 import { zergQueryTool } from './zerg.js';
 import { searchTool } from './search.js';
 import { listSkillsTool } from './skills.js';
+import { screenshotTool, listWindowsTool, runAndMonitorTool } from './screenshot.js';
 
 // --- Tool Registry ---
 
@@ -15,7 +16,10 @@ export const defaultTools: Tool[] = [
   searchTool,
   listSkillsTool,
   runCommandTool,
-  zergQueryTool
+  zergQueryTool,
+  screenshotTool,
+  listWindowsTool,
+  runAndMonitorTool
 ];
 
 export function getToolDefinitions(tools: Tool[] = defaultTools): ToolDefinition[] {
@@ -44,4 +48,5 @@ export { searchTool } from './search.js';
 export { listSkillsTool } from './skills.js';
 export { runCommandTool } from './shell.js';
 export { zergQueryTool } from './zerg.js';
+export { screenshotTool, listWindowsTool, runAndMonitorTool } from './screenshot.js';
 export type { Tool } from './types.js';