twinclaw 1.0.0

package/dist/api/websocket-hub.js

@@ -0,0 +1,305 @@
+import { randomUUID } from 'node:crypto';
+import { WebSocketServer, WebSocket } from 'ws';
+import { logThought } from '../utils/logger.js';
+import { WsCloseCode } from '../types/websocket.js';
+import { getSecretVaultService } from '../services/secret-vault.js';
+// ── Constants ──────────────────────────────────────────────────────────────────
+const DEFAULT_AUTH_TIMEOUT_MS = 5_000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000;
+/** Default backpressure threshold in kilobytes (see WsHubConfig.maxClientQueue). */
+const DEFAULT_MAX_CLIENT_QUEUE = 200;
+const VALID_TOPICS = new Set(['health', 'reliability', 'incidents', 'routing']);
+// ── WsHub ──────────────────────────────────────────────────────────────────────
+/**
+ * Control-plane WebSocket hub.
+ *
+ * Responsibilities:
+ *  - Authenticate clients via a shared-secret handshake within a configurable timeout.
+ *  - Maintain a registry of authenticated, subscribed connections.
+ *  - Fan-out typed event envelopes to clients subscribed to a given topic.
+ *  - Enforce per-client bounded send queues to prevent slow-consumer back-pressure.
+ *  - Run a ping/pong heartbeat and evict stale (unresponsive) clients.
+ *  - Expose operator-facing diagnostics metrics.
+ *  - Invoke an optional `onSubscribe` callback so the event producer can push an
+ *    initial state snapshot immediately after a client subscribes.
+ */
+export class WsHub {
+    #config;
+    #clients = new Map();
+    #wss = null;
+    #heartbeatTimer = null;
+    #seq = 0;
+    #metrics = {
+        totalConnections: 0,
+        authFailures: 0,
+        droppedEvents: 0,
+        staleCleaned: 0,
+        lastEventAt: null,
+    };
+    /**
+     * Called when a client successfully subscribes to topics.
+     * The event producer uses this to dispatch an initial snapshot.
+     */
+    onSubscribe = null;
+    constructor(config = {}) {
+        this.#config = {
+            authTimeoutMs: config.authTimeoutMs ?? DEFAULT_AUTH_TIMEOUT_MS,
+            heartbeatIntervalMs: config.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS,
+            maxClientQueue: config.maxClientQueue ?? DEFAULT_MAX_CLIENT_QUEUE,
+        };
+    }
+    // ── Lifecycle ──────────────────────────────────────────────────────────────
+    /**
+     * Attach the WebSocket server to an existing HTTP server.
+     * Must be called before the HTTP server starts accepting connections.
+     */
+    attach(server) {
+        this.#wss = new WebSocketServer({ server, path: '/ws' });
+        this.#wss.on('connection', (ws, req) => {
+            this.#handleConnection(ws, req);
+        });
+        this.#heartbeatTimer = setInterval(() => {
+            this.#runHeartbeat();
+        }, this.#config.heartbeatIntervalMs);
+        void logThought('[WsHub] WebSocket control-plane attached on /ws.');
+    }
+    /** Gracefully close all connections and shut down the hub. */
+    stop() {
+        if (this.#heartbeatTimer) {
+            clearInterval(this.#heartbeatTimer);
+            this.#heartbeatTimer = null;
+        }
+        for (const client of this.#clients.values()) {
+            this.#closeClient(client, WsCloseCode.ServerShutdown, 'Server shutting down.');
+        }
+        this.#clients.clear();
+        this.#wss?.close();
+        this.#wss = null;
+        void logThought('[WsHub] WebSocket hub stopped.');
+    }
+    // ── Publishing ─────────────────────────────────────────────────────────────
+    /**
+     * Publish a runtime event to all authenticated clients subscribed to `topic`.
+     * Drops the oldest queued message for a client whose send buffer is full.
+     */
+    publish(topic, payload) {
+        const seq = ++this.#seq;
+        const envelope = JSON.stringify({
+            type: 'event',
+            v: 1,
+            topic,
+            seq,
+            ts: new Date().toISOString(),
+            payload,
+        });
+        this.#metrics.lastEventAt = new Date().toISOString();
+        for (const client of this.#clients.values()) {
+            if (!client.authenticated || !client.subscriptions.has(topic))
+                continue;
+            if (client.ws.readyState !== WebSocket.OPEN)
+                continue;
+            this.#sendRaw(client, envelope);
+        }
+    }
+    /**
+     * Send a full-state snapshot directly to a specific client.
+     * Used for the initial snapshot immediately after subscription.
+     */
+    sendSnapshotTo(clientId, snapshot) {
+        const client = this.#clients.get(clientId);
+        if (!client || !client.authenticated || client.ws.readyState !== WebSocket.OPEN)
+            return;
+        const msg = JSON.stringify({
+            type: 'snapshot',
+            v: 1,
+            ts: new Date().toISOString(),
+            ...snapshot,
+        });
+        this.#sendRaw(client, msg);
+    }
+    // ── Diagnostics ────────────────────────────────────────────────────────────
+    /** Returns operator-facing connection and reliability metrics. */
+    getMetrics() {
+        return {
+            activeClients: this.#clients.size,
+            totalConnections: this.#metrics.totalConnections,
+            authFailures: this.#metrics.authFailures,
+            droppedEvents: this.#metrics.droppedEvents,
+            staleCleaned: this.#metrics.staleCleaned,
+            lastEventAt: this.#metrics.lastEventAt,
+        };
+    }
+    // ── Connection Handling ─────────────────────────────────────────────────────
+    #handleConnection(ws, _req) {
+        const clientId = randomUUID();
+        this.#metrics.totalConnections++;
+        const client = {
+            id: clientId,
+            ws,
+            authenticated: false,
+            subscriptions: new Set(),
+            authTimer: null,
+            isAlive: true,
+            connectedAt: Date.now(),
+        };
+        this.#clients.set(clientId, client);
+        // Enforce auth within timeout window
+        client.authTimer = setTimeout(() => {
+            if (!client.authenticated) {
+                this.#metrics.authFailures++;
+                void logThought(`[WsHub] Client ${clientId} auth timeout — closing connection.`);
+                this.#closeClient(client, WsCloseCode.AuthRequired, 'Authentication required.');
+            }
+        }, this.#config.authTimeoutMs);
+        ws.on('pong', () => {
+            client.isAlive = true;
+        });
+        ws.on('message', (data) => {
+            this.#handleMessage(client, data);
+        });
+        ws.on('close', () => {
+            this.#cleanupClient(client);
+        });
+        ws.on('error', (err) => {
+            void logThought(`[WsHub] Client ${clientId} socket error: ${err.message}`);
+            this.#cleanupClient(client);
+        });
+        void logThought(`[WsHub] New connection: ${clientId}.`);
+    }
+    #handleMessage(client, rawData) {
+        let msg;
+        try {
+            msg = JSON.parse(String(rawData));
+        }
+        catch {
+            this.#sendError(client, 400, 'Invalid JSON.');
+            return;
+        }
+        if (typeof msg !== 'object' || msg === null || typeof msg.type !== 'string') {
+            this.#sendError(client, 400, 'Malformed message: missing "type" field.');
+            return;
+        }
+        switch (msg.type) {
+            case 'auth':
+                this.#handleAuth(client, msg);
+                break;
+            case 'subscribe':
+                this.#handleSubscribe(client, msg);
+                break;
+            case 'ping':
+                this.#sendRaw(client, JSON.stringify({ type: 'pong', ts: new Date().toISOString() }));
+                break;
+            default:
+                this.#sendError(client, 400, `Unknown message type: ${String(msg.type)}`);
+        }
+    }
+    #handleAuth(client, msg) {
+        const token = typeof msg.token === 'string' ? msg.token : '';
+        const apiSecret = getSecretVaultService().readSecret('API_SECRET') ?? '';
+        if (!apiSecret || !token || token !== apiSecret) {
+            this.#metrics.authFailures++;
+            void logThought(`[WsHub] Client ${client.id} authentication failed (invalid token).`);
+            this.#sendError(client, WsCloseCode.AuthFailed, 'Authentication failed.');
+            this.#closeClient(client, WsCloseCode.AuthFailed, 'Authentication failed.');
+            return;
+        }
+        if (client.authTimer) {
+            clearTimeout(client.authTimer);
+            client.authTimer = null;
+        }
+        client.authenticated = true;
+        this.#sendRaw(client, JSON.stringify({ type: 'auth_ok', clientId: client.id, ts: new Date().toISOString() }));
+        void logThought(`[WsHub] Client ${client.id} authenticated.`);
+    }
+    #handleSubscribe(client, msg) {
+        if (!client.authenticated) {
+            this.#sendError(client, WsCloseCode.AuthRequired, 'Not authenticated.');
+            return;
+        }
+        const rawTopics = Array.isArray(msg.topics) ? msg.topics : [];
+        const validTopics = [];
+        const invalidTopics = [];
+        for (const t of rawTopics) {
+            if (typeof t === 'string' && VALID_TOPICS.has(t)) {
+                validTopics.push(t);
+            }
+            else {
+                invalidTopics.push(String(t));
+            }
+        }
+        if (invalidTopics.length > 0) {
+            this.#sendError(client, WsCloseCode.InvalidSubscription, `Invalid topics: ${invalidTopics.join(', ')}. Valid topics: ${[...VALID_TOPICS].join(', ')}.`);
+            if (validTopics.length === 0)
+                return;
+        }
+        for (const topic of validTopics) {
+            client.subscriptions.add(topic);
+        }
+        this.#sendRaw(client, JSON.stringify({ type: 'subscribed', topics: validTopics, ts: new Date().toISOString() }));
+        void logThought(`[WsHub] Client ${client.id} subscribed to [${validTopics.join(', ')}].`);
+        if (this.onSubscribe && validTopics.length > 0) {
+            this.onSubscribe(client.id, validTopics);
+        }
+    }
+    // ── Heartbeat ──────────────────────────────────────────────────────────────
+    #runHeartbeat() {
+        for (const client of [...this.#clients.values()]) {
+            if (!client.isAlive) {
+                void logThought(`[WsHub] Client ${client.id} did not respond to ping — evicting stale connection.`);
+                this.#metrics.staleCleaned++;
+                this.#closeClient(client, WsCloseCode.StaleConnection, 'Stale connection.');
+                continue;
+            }
+            client.isAlive = false;
+            try {
+                client.ws.ping();
+            }
+            catch {
+                // If ping fails the close/error event will clean up
+            }
+        }
+    }
+    // ── Helpers ────────────────────────────────────────────────────────────────
+    #closeClient(client, code, reason) {
+        if (client.authTimer) {
+            clearTimeout(client.authTimer);
+            client.authTimer = null;
+        }
+        try {
+            client.ws.close(code, reason);
+        }
+        catch {
+            // ignore — socket may already be closed
+        }
+        this.#clients.delete(client.id);
+    }
+    #cleanupClient(client) {
+        if (client.authTimer) {
+            clearTimeout(client.authTimer);
+            client.authTimer = null;
+        }
+        this.#clients.delete(client.id);
+        void logThought(`[WsHub] Client ${client.id} disconnected.`);
+    }
+    #sendRaw(client, data) {
+        if (client.ws.readyState !== WebSocket.OPEN) {
+            return;
+        }
+        // Per-client backpressure: if the socket's internal buffer is large,
+        // record a dropped event rather than allowing unbounded queue growth.
+        if (client.ws.bufferedAmount > this.#config.maxClientQueue * 1024) {
+            this.#metrics.droppedEvents++;
+            void logThought(`[WsHub] Client ${client.id} backpressure limit hit — dropping event.`);
+            return;
+        }
+        try {
+            client.ws.send(data);
+        }
+        catch {
+            this.#metrics.droppedEvents++;
+        }
+    }
+    #sendError(client, code, message) {
+        this.#sendRaw(client, JSON.stringify({ type: 'error', code, message, ts: new Date().toISOString() }));
+    }
+}

package/dist/config/config-loader.js

@@ -0,0 +1,2 @@
+export { DEFAULT_CONFIG, getConfigPath, ensureConfigDir, readConfig as loadTwinBotJson, readConfig, writeConfig, getConfigValue, reloadConfigSync as reloadConfig, checkAndMigrateWorkspace, hasLegacyConfig, migrateLegacyConfig, } from './json-config.js';
+export { getProfileName, getWorkspaceDir, getWorkspaceSubdir, getDatabasePath, getIdentityDir, getSecretsVaultPath, getTranscriptsDir, ensureWorkspaceDir, ensureWorkspaceSubdirs, initializeWorkspace, initializeWorkspaceGitignore, getWorkspaceSummary, WORKSPACE_GITIGNORE, } from './workspace.js';

package/dist/config/env-schema.js

@@ -0,0 +1,202 @@
+/**
+ * Centralized registry of all environment and secret keys consumed by TwinBot.
+ *
+ * Each entry declares:
+ *   - `key`         The exact env variable name.
+ *   - `type`        Whether the value is a sensitive secret or a plain env var.
+ *   - `class`       'required' | 'optional' | 'conditional'.
+ *   - `scope`       Subsystem that owns the key.
+ *   - `condition`   Feature gate that makes a conditional key applicable.
+ *   - `description` Human-readable purpose.
+ *   - `remediation` Actionable hint when the key is missing or invalid.
+ */
+/**
+ * Complete inventory of environment and secret keys for TwinBot.
+ * Consumed by `EnvValidator` for startup/doctor/health diagnostics.
+ */
+export const CONFIG_SCHEMA = [
+    // ── Runtime Core ────────────────────────────────────────────────────────────
+    {
+        key: 'API_SECRET',
+        type: 'secret',
+        class: 'required',
+        scope: 'runtime',
+        description: 'Master HMAC secret for webhook signature verification. Also used as the secret-vault encryption fallback when SECRET_VAULT_MASTER_KEY is unset.',
+        remediation: "Set API_SECRET in your .env file or register it via `secret set API_SECRET <value>` before starting.",
+    },
+    {
+        key: 'SECRET_VAULT_MASTER_KEY',
+        type: 'env',
+        class: 'optional',
+        scope: 'runtime',
+        description: 'Dedicated AES-256 encryption key for the secret vault. Falls back to API_SECRET if unset.',
+        remediation: 'Optionally set SECRET_VAULT_MASTER_KEY for a dedicated vault encryption key independent of API_SECRET.',
+    },
+    {
+        key: 'SECRET_VAULT_REQUIRED',
+        type: 'env',
+        class: 'optional',
+        scope: 'runtime',
+        description: 'Comma-separated list of additional secret names that must be present at startup.',
+        remediation: 'Set SECRET_VAULT_REQUIRED to enforce presence of extra secrets, e.g. SECRET_VAULT_REQUIRED=DB_PASSWORD,SOME_TOKEN.',
+    },
+    {
+        key: 'API_PORT',
+        type: 'env',
+        class: 'optional',
+        scope: 'runtime',
+        description: 'Listening port for the HTTP control plane API (default: 3100).',
+        remediation: 'Set API_PORT to change the control plane port, e.g. API_PORT=8080.',
+    },
+    {
+        key: 'TOOLS_ALLOW',
+        type: 'env',
+        class: 'optional',
+        scope: 'runtime',
+        description: 'Comma-separated allow-list selectors for tool exposure (exact tool names, group:*, source:*, mcp:<serverId>).',
+        remediation: 'Set TOOLS_ALLOW to restrict model-visible tools, e.g. TOOLS_ALLOW=group:fs,fs.apply_patch.',
+    },
+    {
+        key: 'TOOLS_DENY',
+        type: 'env',
+        class: 'optional',
+        scope: 'runtime',
+        description: 'Comma-separated deny-list selectors for tool exposure, applied after TOOLS_ALLOW.',
+        remediation: 'Set TOOLS_DENY to hide risky tools, e.g. TOOLS_DENY=runtime.exec,group:runtime.',
+    },
+    // ── Model Routing ────────────────────────────────────────────────────────────
+    {
+        key: 'MODAL_API_KEY',
+        type: 'secret',
+        class: 'conditional',
+        condition: 'model:primary',
+        scope: 'model',
+        description: 'API key for the Modal-hosted primary model (GLM-5-FP8).',
+        remediation: 'Set MODAL_API_KEY to enable the primary model provider. At least one model API key is required for AI functionality.',
+    },
+    {
+        key: 'OPENROUTER_API_KEY',
+        type: 'secret',
+        class: 'conditional',
+        condition: 'model:fallback_1',
+        scope: 'model',
+        description: 'API key for the OpenRouter fallback model (stepfun/step-3.5-flash).',
+        remediation: 'Set OPENROUTER_API_KEY to enable OpenRouter as a fallback model provider.',
+    },
+    {
+        key: 'GEMINI_API_KEY',
+        type: 'secret',
+        class: 'conditional',
+        condition: 'model:fallback_2',
+        scope: 'model',
+        description: 'API key for the Google Gemini fallback model (gemini-flash-lite-latest).',
+        remediation: 'Set GEMINI_API_KEY to enable Gemini as a secondary fallback model provider.',
+    },
+    // ── Messaging: Telegram ──────────────────────────────────────────────────────
+    {
+        key: 'TELEGRAM_BOT_TOKEN',
+        type: 'secret',
+        class: 'conditional',
+        condition: 'messaging:telegram',
+        scope: 'messaging',
+        description: 'Telegram Bot API token obtained from @BotFather.',
+        remediation: 'Set TELEGRAM_BOT_TOKEN to enable Telegram messaging. Create a bot at https://t.me/BotFather.',
+    },
+    {
+        key: 'TELEGRAM_USER_ID',
+        type: 'env',
+        class: 'optional',
+        scope: 'messaging',
+        description: 'Optional Telegram user ID allowlist seed (integer) for pre-authorizing yourself before pairing approvals.',
+        remediation: 'Optional: set TELEGRAM_USER_ID to your numeric Telegram user ID. Find it by messaging @userinfobot on Telegram.',
+    },
+    // ── Messaging: WhatsApp ──────────────────────────────────────────────────────
+    {
+        key: 'WHATSAPP_PHONE_NUMBER',
+        type: 'secret',
+        class: 'conditional',
+        condition: 'messaging:whatsapp',
+        scope: 'messaging',
+        description: 'Phone number for the WhatsApp native client (whatsapp-web.js). E.164 format recommended.',
+        remediation: 'Set WHATSAPP_PHONE_NUMBER to enable WhatsApp messaging integration.',
+    },
+    // ── Voice / Audio (Groq) ─────────────────────────────────────────────────────
+    {
+        key: 'GROQ_API_KEY',
+        type: 'secret',
+        class: 'conditional',
+        condition: 'messaging:voice',
+        scope: 'messaging',
+        description: 'Groq API key for Speech-to-Text (Whisper) and Text-to-Speech (Orpheus). Required when Telegram or WhatsApp voice messaging is active.',
+        remediation: 'Set GROQ_API_KEY. A free-tier key is available at https://console.groq.com.',
+    },
+    // ── Embedding ────────────────────────────────────────────────────────────────
+    {
+        key: 'EMBEDDING_PROVIDER',
+        type: 'env',
+        class: 'optional',
+        scope: 'integration',
+        description: 'Embedding backend: "openai" (default) or "ollama" for local-only operation.',
+        remediation: 'Set EMBEDDING_PROVIDER=ollama to use Ollama for embeddings without any API key.',
+    },
+    {
+        key: 'EMBEDDING_API_KEY',
+        type: 'secret',
+        class: 'conditional',
+        condition: 'embedding:openai',
+        scope: 'integration',
+        description: 'API key for the OpenAI-compatible embedding endpoint. Takes priority over OPENAI_API_KEY.',
+        remediation: 'Set EMBEDDING_API_KEY (or OPENAI_API_KEY) to enable remote embedding with an OpenAI-compatible provider.',
+    },
+    {
+        key: 'OPENAI_API_KEY',
+        type: 'secret',
+        class: 'conditional',
+        condition: 'embedding:openai_fallback',
+        scope: 'integration',
+        description: 'OpenAI API key used as a fallback for embeddings when EMBEDDING_API_KEY is unset.',
+        remediation: 'Set OPENAI_API_KEY as an alternative to EMBEDDING_API_KEY for OpenAI embedding access.',
+    },
+    {
+        key: 'EMBEDDING_API_URL',
+        type: 'env',
+        class: 'optional',
+        scope: 'integration',
+        description: 'Custom OpenAI-compatible embedding endpoint URL (default: https://api.openai.com/v1/embeddings).',
+        remediation: 'Override EMBEDDING_API_URL to point at a self-hosted or alternative embedding API.',
+    },
+    {
+        key: 'EMBEDDING_MODEL',
+        type: 'env',
+        class: 'optional',
+        scope: 'integration',
+        description: 'Model name used for OpenAI-compatible embeddings (default: text-embedding-3-small).',
+        remediation: 'Set EMBEDDING_MODEL to use a different OpenAI-compatible embedding model.',
+    },
+    {
+        key: 'MEMORY_EMBEDDING_DIM',
+        type: 'env',
+        class: 'optional',
+        scope: 'storage',
+        description: 'Expected embedding vector dimensionality for sqlite-vec storage (default: 1536). Must match the chosen embedding model.',
+        remediation: "Set MEMORY_EMBEDDING_DIM to match your embedding model's output dimensions to avoid vector shape mismatches.",
+    },
+    {
+        key: 'OLLAMA_BASE_URL',
+        type: 'env',
+        class: 'optional',
+        scope: 'integration',
+        description: 'Base URL for a local Ollama server (default: http://localhost:11434).',
+        remediation: 'Set OLLAMA_BASE_URL if your Ollama instance runs on a non-default host or port.',
+    },
+    {
+        key: 'OLLAMA_EMBEDDING_MODEL',
+        type: 'env',
+        class: 'optional',
+        scope: 'integration',
+        description: 'Ollama model name used for local embeddings (default: mxbai-embed-large).',
+        remediation: 'Set OLLAMA_EMBEDDING_MODEL to override the local Ollama embedding model.',
+    },
+];
+/** Quick lookup map by key name for O(1) resolution. */
+export const CONFIG_SCHEMA_MAP = new Map(CONFIG_SCHEMA.map((spec) => [spec.key, spec]));