visionclaw 0.1.0

package/README.md

@@ -0,0 +1,116 @@
+# VisionClaw
+
+A personal assistant agent that runs on your desktop (macOS/Windows). It receives command messages from pre-configured channels (Gmail, Telegram, Discord) and executes tasks autonomously using desktop control and browser automation. Results are sent back through the same channel.
+
+## Features
+
+- **Autonomous Desktop Agent**: Runs continuously as a long-running process on your computer
+- **Gmail Identity**: The agent has its own Gmail account for email and Google Calendar
+- **Multi-Channel Support**: Receives commands via Gmail, Telegram, Discord
+- **Desktop Control**: Takes screenshots, controls mouse/keyboard, runs terminal commands
+- **Browser Automation**: Navigate and interact with web pages via Playwright
+- **Google Calendar**: Manages its own schedule for recurring tasks and reminders
+- **Self-Improving**: Can add new skills to itself and upgrade to new versions
+- **Runtime Observability**: Built-in HTTP obs page for live logs while the agent is running
+
+## Architecture
+
+VisionClaw is built on the [Claude Agent SDK V2](https://platform.claude.com/docs/en/agent-sdk/typescript-v2-preview). It runs as a single-threaded agent with a wake/sleep loop triggered by incoming messages or a periodic heartbeat.
+
+```
+src/
+  index.ts              # CLI entry point
+  onboarding/           # Interactive setup wizard (Gmail, OAuth, channels)
+  agent/                # Core agent loop, session management, context
+  tools/                # Custom tools (notify, browser, calendar, screenshot, etc.)
+  channels/             # Channel adapters (Gmail, Telegram, Discord)
+  calendar/             # Google Calendar integration
+  memory/               # Persistent memory store
+  config/               # Configuration management
+  obs/                  # Runtime observability HTTP server
+gui/                    # Electron desktop GUI
+```
+
+## Prerequisites
+
+- Node.js >= 20
+- An Anthropic API key
+- A dedicated Gmail account for the agent
+- Google Cloud OAuth2 credentials (Client ID + Secret) with Gmail and Calendar API enabled
+
+## Setup
+
+```bash
+# Install dependencies
+pnpm install
+
+# Build
+pnpm run build
+
+# Run (starts onboarding if not configured)
+pnpm start
+
+# Or run in development mode
+pnpm run dev
+```
+
+The first run triggers an interactive onboarding wizard that will:
+1. Ask for your Anthropic API key
+2. Ask for a dedicated Gmail address for the agent
+3. Walk through Google OAuth2 authorization (Gmail + Calendar scopes)
+4. Optionally configure Telegram and Discord
+
+Configuration is stored per profile at `~/.visionclaw/profiles/<profile>/config.json`.
+
+## Observability (HTTP)
+
+When the agent is running, it serves a local observability page showing live logs.
+
+- URL: `http://127.0.0.1:3101/obs`
+- SSE stream: `GET /obs/events`
+- Snapshot: `GET /obs/snapshot`
+
+This is controlled via **advanced config** (not asked during onboarding):
+
+```json
+{
+  "obs": {
+    "enabled": true,
+    "host": "127.0.0.1",
+    "port": 3101,
+    "bufferSize": 1000
+  }
+}
+```
+
+## GUI
+
+A desktop GUI (Electron) is available for configuration and monitoring:
+
+```bash
+cd gui
+pnpm install
+pnpm start
+```
+
+## Channels
+
+| Channel   | Requirements                    | Status      |
+|-----------|----------------------------------|-------------|
+| Gmail     | Gmail account (required)         | Always on   |
+| Telegram  | Bot token from @BotFather        | Optional    |
+| Discord   | Bot token + channel allowlist    | Optional    |
+
+## Custom Tools
+
+| Tool                | Description |
+|---------------------|-------------|
+| `wait`              | Pause execution for a specified duration |
+| `notify_user`       | Send a message back through a channel |
+| `finish`            | Signal task completion, return to sleep |
+| `take_screenshot`   | Capture desktop screenshot |
+| `manage_skills`     | Install/list skills (built-in + user) |
+| `upgrade`           | Check for and install updates |
+| `manage_calendar`   | Manage Google Calendar |
+| `memory`            | View/create/edit files in the agent memory store |
+| `computer_use_*`    | Mouse/keyboard/scroll/drag/screenshot actions |

package/dist/agent/context.d.ts

@@ -0,0 +1,56 @@
+import type { VisionClawConfig } from "../config/types.js";
+import type { CommandMessage } from "../channels/interface.js";
+/**
+ * A content block in a multimodal user message.
+ * Matches the Anthropic Messages API content block types.
+ */
+export type ContentBlock = {
+    type: "text";
+    text: string;
+} | {
+    type: "image";
+    source: {
+        type: "base64";
+        media_type: "image/png";
+        data: string;
+    } | {
+        type: "url";
+        url: string;
+    };
+};
+/**
+ * Screenshot reference: either a public URL (from Google Drive)
+ * or a raw base64 string as fallback.
+ */
+export interface ScreenshotRef {
+    url?: string;
+    base64?: string;
+}
+/**
+ * Build the context message injected on each agent wake.
+ * Returns a multimodal content block array with text + optional image.
+ */
+export declare function buildWakeContext(options: {
+    config: VisionClawConfig;
+    trigger: "heartbeat" | "message";
+    messages: CommandMessage[];
+    calendarSummary: string;
+    screenshot?: ScreenshotRef;
+}): ContentBlock[];
+/**
+ * Build context for an interrupt message — a new user message that arrives
+ * while the agent is already processing. Lighter than a full wake context
+ * (no screenshot or calendar).
+ */
+export declare function buildInterruptContext(options: {
+    messages: CommandMessage[];
+}): ContentBlock[];
+/**
+ * Build a fresh context after compaction. Returns multimodal blocks.
+ */
+export declare function buildPostCompactionContext(options: {
+    config: VisionClawConfig;
+    calendarSummary: string;
+    pendingMessageCount: number;
+}): ContentBlock[];
+//# sourceMappingURL=context.d.ts.map

package/dist/agent/context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/agent/context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE/D;;;GAGG;AACH,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IACE,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EACF;QAAE,IAAI,EAAE,QAAQ,CAAC;QAAC,UAAU,EAAE,WAAW,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GACzD;QAAE,IAAI,EAAE,KAAK,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;CAClC,CAAC;AAEN;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE;IACxC,MAAM,EAAE,gBAAgB,CAAC;IACzB,OAAO,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,aAAa,CAAC;CAC5B,GAAG,YAAY,EAAE,CAuFjB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE;IAC7C,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B,GAAG,YAAY,EAAE,CA0CjB;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE;IAClD,MAAM,EAAE,gBAAgB,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;CAC7B,GAAG,YAAY,EAAE,CAqBjB"}

package/dist/agent/context.js

@@ -0,0 +1,142 @@
+/**
+ * Build the context message injected on each agent wake.
+ * Returns a multimodal content block array with text + optional image.
+ */
+export function buildWakeContext(options) {
+    const { trigger, messages, calendarSummary, screenshot } = options;
+    const now = new Date().toLocaleString();
+    const blocks = [];
+    // --- Main text block ---
+    const parts = [];
+    parts.push(`[Wake Event] trigger=${trigger} time=${now}`);
+    parts.push("");
+    // Calendar summary
+    if (calendarSummary) {
+        parts.push("## Upcoming Calendar Events");
+        parts.push(calendarSummary);
+        parts.push("");
+    }
+    else {
+        parts.push("## Calendar");
+        parts.push("No upcoming events.");
+        parts.push("");
+    }
+    // Queued messages
+    if (messages.length > 0) {
+        parts.push("## New Messages");
+        for (const msg of messages) {
+            parts.push(`- [${msg.channel}] from ${msg.sender} at ${msg.timestamp}: ${msg.text}`);
+            if (msg.attachments.length > 0) {
+                for (const att of msg.attachments) {
+                    const location = att.url ?? att.localPath ?? "(embedded)";
+                    parts.push(`  Attachment [${att.type}]: ${location}${att.filename ? ` (${att.filename})` : ""}`);
+                }
+            }
+        }
+        parts.push("");
+    }
+    else if (trigger === "heartbeat") {
+        parts.push("## Messages");
+        parts.push("No new messages.");
+        parts.push("");
+    }
+    // Screenshot note
+    if (screenshot?.url ?? screenshot?.base64) {
+        parts.push("## Desktop Screenshot");
+        parts.push("A screenshot of the current desktop is attached below.");
+        parts.push("");
+    }
+    parts.push("Review the above context and decide what to do. If there is nothing to do, call the `finish` tool.");
+    blocks.push({ type: "text", text: parts.join("\n") });
+    // --- Message attachment image blocks ---
+    for (const msg of messages) {
+        for (const att of msg.attachments) {
+            if (att.type === "image" && att.url) {
+                blocks.push({
+                    type: "image",
+                    source: { type: "url", url: att.url },
+                });
+            }
+        }
+    }
+    // --- Screenshot image block ---
+    if (screenshot?.url) {
+        blocks.push({
+            type: "image",
+            source: { type: "url", url: screenshot.url },
+        });
+    }
+    else if (screenshot?.base64) {
+        blocks.push({
+            type: "image",
+            source: {
+                type: "base64",
+                media_type: "image/png",
+                data: screenshot.base64,
+            },
+        });
+    }
+    return blocks;
+}
+/**
+ * Build context for an interrupt message — a new user message that arrives
+ * while the agent is already processing. Lighter than a full wake context
+ * (no screenshot or calendar).
+ */
+export function buildInterruptContext(options) {
+    const { messages } = options;
+    const now = new Date().toLocaleString();
+    const blocks = [];
+    const parts = [];
+    parts.push(`[Interrupt] New message(s) received at ${now} while you are working.`);
+    parts.push("");
+    for (const msg of messages) {
+        parts.push(`- [${msg.channel}] from ${msg.sender} at ${msg.timestamp}: ${msg.text}`);
+        if (msg.attachments.length > 0) {
+            for (const att of msg.attachments) {
+                const location = att.url ?? att.localPath ?? "(embedded)";
+                parts.push(`  Attachment [${att.type}]: ${location}${att.filename ? ` (${att.filename})` : ""}`);
+            }
+        }
+    }
+    parts.push("");
+    parts.push("Decide whether to acknowledge, switch tasks, or continue your current work.");
+    blocks.push({ type: "text", text: parts.join("\n") });
+    // Attach images from the new messages
+    for (const msg of messages) {
+        for (const att of msg.attachments) {
+            if (att.type === "image" && att.url) {
+                blocks.push({
+                    type: "image",
+                    source: { type: "url", url: att.url },
+                });
+            }
+        }
+    }
+    return blocks;
+}
+/**
+ * Build a fresh context after compaction. Returns multimodal blocks.
+ */
+export function buildPostCompactionContext(options) {
+    const { config, calendarSummary, pendingMessageCount } = options;
+    const now = new Date().toLocaleString();
+    return [
+        {
+            type: "text",
+            text: `[Post-Compaction State Refresh] time=${now}
+
+Your identity: ${config.agentName} <${config.gmail}>
+Platform: ${process.platform}
+
+## Calendar
+${calendarSummary || "No upcoming events."}
+
+## Message Queue
+${pendingMessageCount > 0 ? `${pendingMessageCount} message(s) pending in queue.` : "No pending messages."}
+
+Continue from where you left off based on the compaction summary above.`,
+        },
+    ];
+}
+//# sourceMappingURL=context.js.map

package/dist/agent/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/agent/context.ts"],"names":[],"mappings":"AAyBA;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAMhC;IACC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACnE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,cAAc,EAAE,CAAC;IAExC,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,0BAA0B;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,wBAAwB,OAAO,SAAS,GAAG,EAAE,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,mBAAmB;IACnB,IAAI,eAAe,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,kBAAkB;IAClB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACR,MAAM,GAAG,CAAC,OAAO,UAAU,GAAG,CAAC,MAAM,OAAO,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,IAAI,EAAE,CACzE,CAAC;YACF,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;oBAClC,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,IAAI,YAAY,CAAC;oBAC1D,KAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,CAAC,IAAI,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;SAAM,IAAI,OAAO,KAAK,WAAW,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,kBAAkB;IAClB,IAAI,UAAU,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;QACrE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CACR,oGAAoG,CACrG,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEtD,0CAA0C;IAC1C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YAClC,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE;iBACtC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,IAAI,UAAU,EAAE,GAAG,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE;SAC7C,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,WAAW;gBACvB,IAAI,EAAE,UAAU,CAAC,MAAM;aACxB;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAErC;IACC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,cAAc,EAAE,CAAC;IAExC,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,0CAA0C,GAAG,yBAAyB,CAAC,CAAC;IACnF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CACR,MAAM,GAAG,CAAC,OAAO,UAAU,GAAG,CAAC,MAAM,OAAO,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,IAAI,EAAE,CACzE,CAAC;QACF,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;gBAClC,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,IAAI,YAAY,CAAC;gBAC1D,KAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,CAAC,IAAI,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnG,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,6EAA6E,CAC9E,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEtD,sCAAsC;IACtC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YAClC,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE;iBACtC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAI1C;IACC,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC;IACjE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,cAAc,EAAE,CAAC;IAExC,OAAO;QACL;YACE,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,wCAAwC,GAAG;;iBAEtC,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,KAAK;YACtC,OAAO,CAAC,QAAQ;;;EAG1B,eAAe,IAAI,qBAAqB;;;EAGxC,mBAAmB,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,mBAAmB,+BAA+B,CAAC,CAAC,CAAC,sBAAsB;;wEAElC;SACnE;KACF,CAAC;AACJ,CAAC"}

package/dist/agent/loop.d.ts

@@ -0,0 +1,18 @@
+import type { VisionClawConfig } from "../config/types.js";
+import { MessageQueue } from "../channels/queue.js";
+import { ChannelManager } from "../channels/manager.js";
+/** Shared state visible to tools (e.g. notify_user needs channel access) */
+export interface AgentState {
+    config: VisionClawConfig;
+    channelManager: ChannelManager;
+    messageQueue: MessageQueue;
+    busy: boolean;
+    lastMessageChannel?: string;
+    lastMessageSender?: string;
+}
+export declare function getAgentState(): AgentState;
+/**
+ * Main agent loop. This is the entry point that runs forever.
+ */
+export declare function startAgentLoop(config: VisionClawConfig): Promise<void>;
+//# sourceMappingURL=loop.d.ts.map

package/dist/agent/loop.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loop.d.ts","sourceRoot":"","sources":["../../src/agent/loop.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAM3D,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAMxD,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,gBAAgB,CAAC;IACzB,cAAc,EAAE,cAAc,CAAC;IAC/B,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAKD,wBAAgB,aAAa,IAAI,UAAU,CAK1C;AAmBD;;GAEG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,IAAI,CAAC,CAiWf"}

package/dist/agent/loop.js

@@ -0,0 +1,323 @@
+import { buildSystemPrompt } from "./system-prompt.js";
+import { AgentSession } from "./session.js";
+import { buildWakeContext, buildInterruptContext } from "./context.js";
+import { createToolServer } from "../tools/index.js";
+import { MessageQueue } from "../channels/queue.js";
+import { ChannelManager } from "../channels/manager.js";
+import { takeScreenshot } from "../tools/screenshot.js";
+import { getCalendarSummary } from "../calendar/scheduler.js";
+import { initScreenshotUploader, uploadScreenshot } from "../files.js";
+import { logger } from "../logger.js";
+// Module-level state so tools can access it
+let agentState = null;
+export function getAgentState() {
+    if (!agentState) {
+        throw new Error("Agent state not initialized");
+    }
+    return agentState;
+}
+/**
+ * Log an SDK result message (success or error).
+ */
+function logResult(msg) {
+    const cost = msg.total_cost_usd.toFixed(4);
+    logger.result(msg.num_turns, msg.usage.input_tokens, msg.usage.output_tokens, cost);
+    if (msg.is_error) {
+        const errors = "errors" in msg ? msg.errors.join("; ") : msg.subtype;
+        logger.err(`Agent ended with error (${msg.subtype}): ${errors}`);
+    }
+}
+/**
+ * Main agent loop. This is the entry point that runs forever.
+ */
+export async function startAgentLoop(config) {
+    // --- Initialize components ---
+    const messageQueue = new MessageQueue();
+    const channelManager = new ChannelManager(config, messageQueue);
+    agentState = {
+        config,
+        channelManager,
+        messageQueue,
+        busy: false,
+    };
+    // Initialize S3 client for screenshot uploads
+    initScreenshotUploader();
+    // Create the in-process MCP tool server
+    const toolServer = createToolServer();
+    // Build system prompt
+    const systemPrompt = buildSystemPrompt(config);
+    // Create the agent session (uses V1 query API with full tool support)
+    const session = new AgentSession(config, systemPrompt, {
+        visionclaw: toolServer,
+        playwright: {
+            command: "npx",
+            args: [
+                "@playwright/mcp@latest",
+                "--browser",
+                process.platform === "darwin" ? "chrome" : "chromium",
+            ],
+        },
+    });
+    // Start all enabled channels
+    await channelManager.startAll();
+    // Listen for incoming messages to wake the agent or inject interrupts
+    let wakeResolver = null;
+    channelManager.on("message", () => {
+        // If agent is sleeping, wake it up
+        if (wakeResolver !== null) {
+            wakeResolver();
+            wakeResolver = null;
+            return;
+        }
+        // If agent is busy, inject interrupt messages into the active session
+        if (agentState?.busy) {
+            const interruptMessages = [];
+            while (!messageQueue.isEmpty()) {
+                const msg = messageQueue.dequeue();
+                if (msg)
+                    interruptMessages.push(msg);
+            }
+            if (interruptMessages.length > 0) {
+                // Update last message tracking
+                const last = interruptMessages[interruptMessages.length - 1];
+                agentState.lastMessageChannel = last.channel;
+                agentState.lastMessageSender = last.sender;
+                // Log the interrupt
+                for (const msg of interruptMessages) {
+                    logger.log("info", "incoming", `[interrupt] [${msg.channel}] ${msg.sender}: ${msg.text}`, {
+                        channel: msg.channel,
+                        sender: msg.sender,
+                        text: msg.text,
+                        id: msg.id,
+                        interrupt: true,
+                    });
+                }
+                // Build interrupt context and inject into session
+                const interruptBlocks = buildInterruptContext({ messages: interruptMessages });
+                const interruptContent = interruptBlocks;
+                session.injectMessage(interruptContent);
+            }
+        }
+    });
+    logger.system("Agent loop started. Waiting for messages or heartbeat...");
+    // --- Main loop ---
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- intentional infinite loop
+    while (true) {
+        agentState.busy = false;
+        // Sleep until heartbeat or new message
+        await new Promise((resolve) => {
+            wakeResolver = resolve;
+            const timer = setTimeout(() => {
+                if (wakeResolver === resolve) {
+                    wakeResolver = null;
+                    resolve();
+                }
+            }, config.heartbeatIntervalMs);
+            if (!messageQueue.isEmpty()) {
+                clearTimeout(timer);
+                wakeResolver = null;
+                resolve();
+            }
+        });
+        agentState.busy = true;
+        // Determine trigger
+        const hasMessages = !messageQueue.isEmpty();
+        const trigger = hasMessages ? "message" : "heartbeat";
+        // Dequeue all pending messages
+        const messages = [];
+        while (!messageQueue.isEmpty()) {
+            const msg = messageQueue.dequeue();
+            if (msg)
+                messages.push(msg);
+        }
+        // Track last message for notify_user
+        if (messages.length > 0) {
+            const last = messages[messages.length - 1];
+            agentState.lastMessageChannel = last.channel;
+            agentState.lastMessageSender = last.sender;
+        }
+        // Log the wake event
+        logger.wake(trigger, messages.length);
+        // Log each incoming message
+        for (const msg of messages) {
+            logger.log("info", "incoming", `[${msg.channel}] ${msg.sender}: ${msg.text}`, {
+                channel: msg.channel,
+                sender: msg.sender,
+                text: msg.text,
+                id: msg.id,
+            });
+        }
+        // --- Gather context ---
+        // Gather screenshot and calendar in parallel
+        const [screenshotResult, calendarResult] = await Promise.allSettled([
+            (async () => {
+                const t0 = Date.now();
+                const base64 = await takeScreenshot();
+                logger.debug(`Screenshot captured in ${Date.now() - t0}ms`);
+                const t1 = Date.now();
+                const url = await uploadScreenshot(base64);
+                logger.debug(`Screenshot uploaded in ${Date.now() - t1}ms`);
+                return url ? { url } : { base64 };
+            })(),
+            (async () => {
+                const t0 = Date.now();
+                const summary = await getCalendarSummary(config);
+                logger.debug(`Calendar fetched in ${Date.now() - t0}ms`);
+                return summary;
+            })(),
+        ]);
+        const screenshot = screenshotResult.status === "fulfilled" ? screenshotResult.value : undefined;
+        if (screenshotResult.status === "rejected") {
+            logger.warn(`Screenshot failed: ${screenshotResult.reason instanceof Error ? screenshotResult.reason.message : String(screenshotResult.reason)}`);
+        }
+        const calendarSummary = calendarResult.status === "fulfilled" ? calendarResult.value : "";
+        if (calendarResult.status === "rejected") {
+            logger.warn(`Calendar fetch failed: ${calendarResult.reason instanceof Error ? calendarResult.reason.message : String(calendarResult.reason)}`);
+        }
+        // Build multimodal content blocks
+        const contentBlocks = buildWakeContext({
+            config,
+            trigger,
+            messages,
+            calendarSummary,
+            screenshot,
+        });
+        // Map tool_use_id -> tool name for correlating results (per wake cycle)
+        const toolNameById = new Map();
+        // Log the multimodal message being sent (text + image summary)
+        logger.sendMultimodal(contentBlocks);
+        try {
+            // Send the multimodal content and stream the response.
+            // Cast through unknown because our ContentBlock type may not
+            // exactly match MessageParam["content"].
+            const messageContent = contentBlocks;
+            const queryStream = session.sendAndStream(messageContent);
+            // Process the response stream.
+            // We don't break on finish -- breaking calls generator.return() which
+            // can hang waiting for the SDK child process to exit. Instead we
+            // drain the stream naturally and skip logging after finish.
+            let finishCalled = false;
+            for await (const msg of queryStream) {
+                // Always capture session ID
+                if ("session_id" in msg && msg.session_id) {
+                    session.captureSessionId(msg.session_id);
+                }
+                // After finish, only log the result message and skip everything else
+                if (finishCalled) {
+                    if (msg.type === "result") {
+                        logResult(msg);
+                    }
+                    continue;
+                }
+                // Handle different message types
+                switch (msg.type) {
+                    case "user": {
+                        // In Claude Agent SDK v2, tool execution results are surfaced back
+                        // into the stream as "user" messages (not as a separate tool_result
+                        // message type). Capture and log them for OBS.
+                        const um = msg;
+                        // Tool results in this SDK are encoded as user message content blocks:
+                        // [{ type: "tool_result", tool_use_id, content: [...] }]
+                        const blocks = um.message?.content;
+                        if (Array.isArray(blocks)) {
+                            for (const b of blocks) {
+                                const block = b;
+                                if (block.type !== "tool_result" || !block.tool_use_id)
+                                    continue;
+                                const out = block.content;
+                                const summary = out === undefined
+                                    ? ""
+                                    : typeof out === "string"
+                                        ? out
+                                        : JSON.stringify(out);
+                                const resolvedName = toolNameById.get(block.tool_use_id) ?? "tool";
+                                logger.toolResult(resolvedName, summary, {
+                                    tool_use_id: block.tool_use_id,
+                                    is_error: block.is_error,
+                                    source: "sdk_user_message_block",
+                                });
+                            }
+                        }
+                        break;
+                    }
+                    case "assistant": {
+                        const assistantMsg = msg;
+                        const content = assistantMsg.message.content;
+                        for (const block of content) {
+                            switch (block.type) {
+                                case "text":
+                                    if (block.text) {
+                                        logger.assistant(block.text);
+                                    }
+                                    break;
+                                case "thinking":
+                                    logger.debug(`[thinking] ${block.thinking.substring(0, 200)}${block.thinking.length > 200 ? "..." : ""}`);
+                                    break;
+                                case "tool_use":
+                                    {
+                                        const toolUseId = block.id ??
+                                            block.tool_use_id;
+                                        if (toolUseId) {
+                                            toolNameById.set(toolUseId, block.name);
+                                        }
+                                        logger.toolCall(block.name, block.input, {
+                                            tool_use_id: toolUseId,
+                                        });
+                                    }
+                                    if (block.name === "finish" || block.name === "mcp__visionclaw__finish") {
+                                        finishCalled = true;
+                                        session.closeInput();
+                                    }
+                                    break;
+                                default:
+                                    // Log other block types (server_tool_use, mcp_tool_use, etc.)
+                                    logger.debug(`Assistant block: ${block.type}`);
+                                    break;
+                            }
+                        }
+                        // Log assistant-level errors
+                        if (assistantMsg.error) {
+                            logger.err(`Assistant error: ${assistantMsg.error}`);
+                        }
+                        break;
+                    }
+                    case "result":
+                        logResult(msg);
+                        break;
+                    case "system":
+                        logger.debug(`System message: ${JSON.stringify(msg)}`, {
+                            subtype: msg.subtype,
+                        });
+                        break;
+                    default:
+                        // stream_event, status, task_notification, etc. — skip silently
+                        // Also handle tool_result even if the SDK type union doesn't include it.
+                        if (msg.type === "tool_result") {
+                            const tr = msg;
+                            const name = tr.name ?? "tool";
+                            const summary = tr.content === undefined
+                                ? ""
+                                : typeof tr.content === "string"
+                                    ? tr.content
+                                    : JSON.stringify(tr.content);
+                            logger.toolResult(name, summary, {
+                                tool_use_id: tr.tool_use_id,
+                                is_error: tr.is_error,
+                            });
+                        }
+                        break;
+                }
+            }
+            // Ensure generator is closed after stream ends
+            session.closeInput();
+        }
+        catch (err) {
+            // Ensure generator is closed on error too
+            session.closeInput();
+            logger.err(`Agent loop error: ${err instanceof Error ? err.message : String(err)}`, { stack: err instanceof Error ? err.stack : undefined });
+            // Wait a bit before retrying to avoid tight error loops
+            await new Promise((r) => setTimeout(r, 5000));
+        }
+    }
+}
+//# sourceMappingURL=loop.js.map