thepopebot 1.2.75-beta.1 → 1.2.75-beta.4

package/README.md

@@ -192,6 +192,17 @@ See [Different Models](docs/RUNNING_DIFFERENT_MODELS.md) for the full provider r
 
 ---
 
+## Known Issues
+
+### Windows: `SQLITE_IOERR_SHMOPEN`
+
+SQLite can't create or open its shared-memory (`.shm`) file. Common causes:
+
+- **Antivirus** (Windows Defender, etc.) locking the database files — add your project folder to the exclusion list
+- **Cloud-synced folders** (OneDrive, Dropbox, Google Drive) — move your project to a non-synced directory like `C:\Projects\`
+
+---
+
 ## Docs
 
 | Document | Description |

package/bin/cli.js

@@ -52,11 +52,13 @@ Commands:
   init                              Scaffold a new thepopebot project
   upgrade|update [@beta|version]    Upgrade thepopebot (install, init, build, commit, push)
   setup                             Run interactive setup wizard
+  setup-ssl                         Configure SSL with Let's Encrypt wildcard cert
   setup-telegram                    Reconfigure Telegram webhook
   reset-auth                        Regenerate AUTH_SECRET (invalidates all sessions)
   reset [file]                      Restore a template file (or list available templates)
   diff [file]                       Show differences between project files and package templates
   sync <path>                       Sync local package to a test install (build, pack, Docker)
+  sync --fast <path>                Fast sync — copy source into running container, rebuild .next
   set-var <KEY> [VALUE]             Set a GitHub repository variable
   user:password <email>             Change a user's password
 `);
@@ -337,8 +339,7 @@ AUTH_TRUST_HOST=true
 DATABASE_PATH=data/db/thepopebot.sqlite
 THEPOPEBOT_VERSION=${version}
 
-# Uncomment to use a custom docker-compose file that won't be overwritten by upgrades.
-# Edit docker-compose.custom.yml with your changes, then uncomment:
+# To enable SSL with Let's Encrypt, run: npx thepopebot setup-ssl
 # COMPOSE_FILE=docker-compose.custom.yml
 `;
     fs.writeFileSync(envPath, seedEnv);
@@ -490,6 +491,15 @@ function setup() {
   }
 }
 
+function setupSsl() {
+  const setupScript = path.join(__dirname, '..', 'setup', 'setup-ssl.mjs');
+  try {
+    execFileSync(process.execPath, [setupScript], { stdio: 'inherit', cwd: process.cwd() });
+  } catch {
+    process.exit(1);
+  }
+}
+
 function setupTelegram() {
   const setupScript = path.join(__dirname, '..', 'setup', 'setup-telegram.mjs');
   try {
@@ -770,6 +780,9 @@ switch (command) {
   case 'setup':
     setup();
     break;
+  case 'setup-ssl':
+    setupSsl();
+    break;
   case 'setup-telegram':
     setupTelegram();
     break;
@@ -787,8 +800,15 @@ switch (command) {
     await upgrade();
     break;
   case 'sync': {
-    const { sync } = await import('./sync.js');
-    await sync(args[0]);
+    const fast = args.includes('--fast');
+    const syncArgs = args.filter(a => a !== '--fast');
+    if (fast) {
+      const { syncFast } = await import('./sync.js');
+      await syncFast(syncArgs[0]);
+    } else {
+      const { sync } = await import('./sync.js');
+      await sync(syncArgs[0]);
+    }
     break;
   }
   case 'set-var':

package/bin/docker-build.js

@@ -62,6 +62,11 @@ const CODING_AGENTS = [
     context: 'docker/coding-agent',
     dockerfile: 'docker/coding-agent/Dockerfile.opencode',
   },
+  {
+    name: 'coding-agent-kimi-cli',
+    context: 'docker/coding-agent',
+    dockerfile: 'docker/coding-agent/Dockerfile.kimi-cli',
+  },
 ];
 
 // Non-coding-agent images (independent, built in parallel)

package/bin/sync.js

@@ -306,6 +306,90 @@ function buildDockerImage(projectPath) {
   }
 }
 
+/**
+ * Fast sync — skip Docker image rebuild entirely.
+ *
+ *   1. Build package JSX (npm run build)
+ *   2. mirrorTemplates() — scaffold using init's managed-path logic
+ *   3. docker cp package source (lib/, api/, config/, package.json) into
+ *      the running container's /app/node_modules/thepopebot/
+ *   4. docker cp web/app/ + web/postcss.config.mjs into container
+ *   5. docker exec next build inside the container (tailwindcss already there)
+ *   6. Clean up copied source from container
+ *   7. docker exec pm2 restart all
+ */
+export async function syncFast(projectPath) {
+  if (!projectPath) {
+    console.error('\n  Usage: thepopebot sync --fast <path-to-project>\n');
+    process.exit(1);
+  }
+
+  projectPath = path.resolve(projectPath);
+
+  if (!fs.existsSync(path.join(projectPath, 'package.json'))) {
+    console.error(`\n  Not a project directory (no package.json): ${projectPath}\n`);
+    process.exit(1);
+  }
+
+  // 1. Build JSX
+  console.log('\n  Building package...');
+  execSync('npm run build', { stdio: 'inherit', cwd: PACKAGE_DIR });
+
+  // 2. Mirror templates
+  console.log('\n  Mirroring templates...');
+  mirrorTemplates(projectPath);
+
+  // 3. Get running container ID
+  const container = execSync('docker compose ps -q event-handler', {
+    encoding: 'utf8',
+    cwd: projectPath,
+  }).trim();
+
+  if (!container) {
+    console.error('\n  event-handler container is not running. Use full sync instead.\n');
+    process.exit(1);
+  }
+
+  // 4. Copy package source into container's node_modules/thepopebot/
+  const PKG_DEST = '/app/node_modules/thepopebot';
+  const PACKAGE_DIRS = ['lib', 'api', 'config'];
+
+  console.log('\n  Copying package source into container...');
+  for (const dir of PACKAGE_DIRS) {
+    execSync(`docker exec ${container} rm -rf ${PKG_DEST}/${dir}`, { stdio: 'inherit' });
+    execSync(`docker cp ${path.join(PACKAGE_DIR, dir)} ${container}:${PKG_DEST}/${dir}`, { stdio: 'inherit' });
+  }
+  // Also copy package.json for exports resolution
+  execSync(`docker cp ${path.join(PACKAGE_DIR, 'package.json')} ${container}:${PKG_DEST}/package.json`, { stdio: 'inherit' });
+
+  // 5. Copy web/app/ source into container for next build
+  const webDir = path.join(PACKAGE_DIR, 'web');
+  console.log('\n  Copying web source into container...');
+  execSync(`docker cp ${path.join(webDir, 'app')} ${container}:/app/app`, { stdio: 'inherit' });
+  execSync(`docker cp ${path.join(webDir, 'postcss.config.mjs')} ${container}:/app/postcss.config.mjs`, { stdio: 'inherit' });
+  execSync(`docker cp ${path.join(webDir, 'next.config.mjs')} ${container}:/app/next.config.mjs`, { stdio: 'inherit' });
+
+  // 6. Run next build inside the container
+  // Hide data/logs dirs so webpack's FileSystemInfo doesn't crawl them (causes OOM/RangeError
+  // when workspaces contain thousands of files). Restored immediately after build.
+  console.log('\n  Building Next.js inside container...');
+  execSync(`docker exec ${container} sh -c 'mv /app/data /app/.data-build-tmp 2>/dev/null; mv /app/logs /app/.logs-build-tmp 2>/dev/null; true'`, { stdio: 'inherit' });
+  try {
+    execSync(`docker exec ${container} ./node_modules/.bin/next build`, { stdio: 'inherit' });
+  } finally {
+    execSync(`docker exec ${container} sh -c 'mv /app/.data-build-tmp /app/data 2>/dev/null; mv /app/.logs-build-tmp /app/logs 2>/dev/null; true'`, { stdio: 'inherit' });
+  }
+
+  // 7. Clean up web source from container (not needed at runtime)
+  execSync(`docker exec ${container} rm -rf /app/app`, { stdio: 'inherit' });
+
+  // 8. Restart PM2
+  console.log('\n  Restarting server...');
+  execSync(`docker exec ${container} pm2 restart all`, { stdio: 'inherit' });
+
+  console.log('\n  Fast synced!\n');
+}
+
 export async function sync(projectPath) {
   if (!projectPath) {
     console.error('\n  Usage: thepopebot sync <path-to-project>\n');

package/lib/ai/async-channel.js

@@ -0,0 +1,51 @@
+/**
+ * Async push/pull queue. Producer calls push()/done(), consumer uses for-await.
+ */
+export function createChannel() {
+  const queue = [];
+  const waiters = [];
+  let isDone = false;
+
+  return {
+    push(value) {
+      if (waiters.length > 0) waiters.shift()(value);
+      else queue.push(value);
+    },
+    done() {
+      isDone = true;
+      while (waiters.length > 0) waiters.shift()(Symbol.for('done'));
+    },
+    async *[Symbol.asyncIterator]() {
+      while (true) {
+        if (queue.length > 0) {
+          yield queue.shift();
+        } else if (isDone) {
+          return;
+        } else {
+          const value = await new Promise(resolve => waiters.push(resolve));
+          if (value === Symbol.for('done')) return;
+          yield value;
+        }
+      }
+    }
+  };
+}
+
+/**
+ * Merge two async iterables — yields from whichever has data first.
+ * Completes when BOTH are exhausted.
+ */
+export async function* mergeAsyncIterables(iter1, iter2) {
+  const channel = createChannel();
+  let active = 2;
+
+  const consume = async (iter) => {
+    for await (const item of iter) channel.push(item);
+    if (--active === 0) channel.done();
+  };
+
+  consume(iter1);
+  consume(iter2);
+
+  yield* channel;
+}

package/lib/ai/index.js

@@ -1,4 +1,5 @@
 import { HumanMessage, AIMessage } from '@langchain/core/messages';
+import { createChannel, mergeAsyncIterables } from './async-channel.js';
 import { z } from 'zod';
 import { getAgentChat, getCodeChat } from './agent.js';
 import { createModel } from './model.js';
@@ -178,16 +179,22 @@ async function* chatStream(threadId, message, attachments = [], options = {}) {
     }
   }
 
+  // Side channel: bridges the tool's live container output to this generator
+  const sideChannel = createChannel();
+  const streamCallback = (chunk) => {
+    if (chunk === null) sideChannel.done();
+    else sideChannel.push(chunk);
+  };
+
   try {
     const stream = await agent.stream(
       { messages: [new HumanMessage({ content: messageContent })] },
-      { configurable: { thread_id: threadId, workspaceId, repo, branch, codeModeType }, streamMode: 'messages' }
+      { configurable: { thread_id: threadId, workspaceId, repo, branch, codeModeType, streamCallback }, streamMode: 'messages' }
     );
 
     let fullText = '';
     const toolCallNames = {};
     const pendingToolCalls = new Map();
-    let headlessContainer = null;
 
     // Accumulate raw tool call arg fragments across streaming chunks.
     // Each AIMessageChunk only carries its own delta — the first chunk
@@ -195,120 +202,127 @@ async function* chatStream(threadId, message, attachments = [], options = {}) {
     // chunks (input_json_delta) have only index with the partial JSON delta.
     const toolCallRawArgs = {};     // tool_call_id → accumulated args string
     const indexToToolCallId = {};   // chunk index → tool_call_id
+    const toolCallArgsEmitted = new Set(); // tool_call_ids whose complete args have been yielded
+
+    // Headless container streaming state
+    const memoryParts = [];
+    const headlessPendingToolCalls = new Map();
+    let pendingText = '';       // channel text, flushed to DB at tool boundaries
+    let llmTextAccum = '';      // langgraph text (direct response or LLM follow-up after container)
+    let resultSummary = '';
+
+    // Tag helper so mergeAsyncIterables can tell the two sources apart.
+    // The LangGraph wrapper also closes sideChannel when the agent stream
+    // finishes — this prevents a deadlock when no tool calls streamCallback.
+    async function* tagged(iter, source) {
+      for await (const item of iter) yield { _src: source, item };
+      if (source === 'lg') sideChannel.done();
+    }
 
-    for await (const event of stream) {
-      // streamMode: 'messages' yields [message, metadata] tuples
-      const msg = Array.isArray(event) ? event[0] : event;
-      const msgType = msg._getType?.();
-
-      if (msgType === 'ai') {
-        // Tool calls — AIMessage.tool_calls is an array of { id, name, args }
-        if (msg.tool_calls?.length > 0) {
-          for (const tc of msg.tool_calls) {
-            toolCallNames[tc.id] = tc.name;
-            pendingToolCalls.set(tc.id, { toolName: tc.name, args: tc.args });
-            yield {
-              type: 'tool-call',
-              toolCallId: tc.id,
-              toolName: tc.name,
-              args: tc.args,
-            };
-          }
-        }
-
-        // Accumulate raw tool call arg strings from streaming chunks
-        if (msg.tool_call_chunks?.length > 0) {
-          for (const c of msg.tool_call_chunks) {
-            if (c.id) {
-              indexToToolCallId[c.index] = c.id;
-              toolCallRawArgs[c.id] = (toolCallRawArgs[c.id] || '') + (c.args || '');
-            } else if (c.index != null && indexToToolCallId[c.index]) {
-              const id = indexToToolCallId[c.index];
-              toolCallRawArgs[id] = (toolCallRawArgs[id] || '') + (c.args || '');
+    try {
+      for await (const { _src, item } of mergeAsyncIterables(
+        tagged(stream, 'lg'),
+        tagged(sideChannel, 'ch')
+      )) {
+        if (_src === 'lg') {
+          // ── LangGraph agent stream ────────────────────────────────────────
+          const msg = Array.isArray(item) ? item[0] : item;
+          const msgType = msg._getType?.();
+
+          if (msgType === 'ai') {
+            // Tool calls — AIMessage.tool_calls is an array of { id, name, args }
+            if (msg.tool_calls?.length > 0) {
+              for (const tc of msg.tool_calls) {
+                toolCallNames[tc.id] = tc.name;
+                pendingToolCalls.set(tc.id, { toolName: tc.name, args: tc.args });
+                yield {
+                  type: 'tool-call',
+                  toolCallId: tc.id,
+                  toolName: tc.name,
+                  args: tc.args,
+                };
+              }
             }
-          }
-        }
 
-        // Text content (wrapped in structured object)
-        let text = '';
-        if (typeof msg.content === 'string') {
-          text = msg.content;
-        } else if (Array.isArray(msg.content)) {
-          text = msg.content
-            .filter((b) => b.type === 'text' && b.text)
-            .map((b) => b.text)
-            .join('');
-        }
-
-        if (text) {
-          fullText += text;
-          yield { type: 'text', text };
-        }
-      } else if (msgType === 'tool') {
-        // Parse complete args from accumulated raw fragments
-        const tc = pendingToolCalls.get(msg.tool_call_id);
-        const rawArgs = toolCallRawArgs[msg.tool_call_id];
-        let completeArgs;
-        try { completeArgs = rawArgs ? JSON.parse(rawArgs) : {}; } catch { completeArgs = {}; }
-
-        // Tool result — ToolMessage has tool_call_id and content
-        yield {
-          type: 'tool-result',
-          toolCallId: msg.tool_call_id,
-          toolName: tc?.toolName,
-          args: completeArgs,
-          result: msg.content,
-        };
-
-        // Save complete tool invocation as JSON
-        if (tc) {
-          persistMessage(threadId, 'assistant', JSON.stringify({
-            type: 'tool-invocation',
-            toolCallId: msg.tool_call_id,
-            toolName: tc.toolName,
-            state: 'output-available',
-            input: completeArgs,
-            output: msg.content,
-          }), options);
-          pendingToolCalls.delete(msg.tool_call_id);
-        }
-
-        // Detect headless container tool result for Phase 2 streaming
-        const headlessToolName = toolCallNames[msg.tool_call_id];
-        if (headlessToolName === 'coding_agent') {
-          try {
-            const parsed = JSON.parse(msg.content);
-            if (parsed.status === 'started' && parsed.containerName) {
-              headlessContainer = { ...parsed, toolName: headlessToolName };
+            // Accumulate raw tool call arg strings from streaming chunks
+            if (msg.tool_call_chunks?.length > 0) {
+              for (const c of msg.tool_call_chunks) {
+                if (c.id) {
+                  indexToToolCallId[c.index] = c.id;
+                  toolCallRawArgs[c.id] = (toolCallRawArgs[c.id] || '') + (c.args || '');
+                } else if (c.index != null && indexToToolCallId[c.index]) {
+                  const id = indexToToolCallId[c.index];
+                  toolCallRawArgs[id] = (toolCallRawArgs[id] || '') + (c.args || '');
+                }
+              }
+              // Re-yield tool-call with complete args once the JSON is fully streamed
+              for (const c of msg.tool_call_chunks) {
+                const id = c.id || indexToToolCallId[c.index];
+                if (id && toolCallRawArgs[id] && !toolCallArgsEmitted.has(id)) {
+                  try {
+                    const parsed = JSON.parse(toolCallRawArgs[id]);
+                    toolCallArgsEmitted.add(id);
+                    const tc = pendingToolCalls.get(id);
+                    if (tc) {
+                      tc.args = parsed;
+                      yield { type: 'tool-call', toolCallId: id, toolName: tc.toolName, args: parsed };
+                    }
+                  } catch {} // args not complete yet, keep accumulating
+                }
+              }
             }
-          } catch {}
-        }
-      }
-      // Skip other message types (human, system)
-    }
 
-    // Save assistant response to DB (defer if headless streaming follows)
-    if (fullText && !headlessContainer) {
-      persistMessage(threadId, 'assistant', fullText, options);
-    }
+            // Text content (wrapped in structured object)
+            let text = '';
+            if (typeof msg.content === 'string') {
+              text = msg.content;
+            } else if (Array.isArray(msg.content)) {
+              text = msg.content
+                .filter((b) => b.type === 'text' && b.text)
+                .map((b) => b.text)
+                .join('');
+            }
 
-    // Phase 2: Stream headless container output live
-    if (headlessContainer) {
-      try {
-        const { tailContainerLogs, waitForContainer, removeContainer } =
-          await import('../tools/docker.js');
-        const { parseHeadlessStream } = await import('./headless-stream.js');
+            if (text) {
+              fullText += text;
+              llmTextAccum += text;
+              yield { type: 'text', text };
+            }
+          } else if (msgType === 'tool') {
+            // Parse complete args from accumulated raw fragments
+            const tc = pendingToolCalls.get(msg.tool_call_id);
+            const rawArgs = toolCallRawArgs[msg.tool_call_id];
+            let completeArgs;
+            try { completeArgs = rawArgs ? JSON.parse(rawArgs) : {}; } catch { completeArgs = {}; }
+
+            // Tool result — ToolMessage has tool_call_id and content
+            yield {
+              type: 'tool-result',
+              toolCallId: msg.tool_call_id,
+              toolName: tc?.toolName,
+              args: completeArgs,
+              result: msg.content,
+            };
 
-        const logStream = await tailContainerLogs(headlessContainer.containerName);
+            // Save complete tool invocation as JSON
+            if (tc) {
+              persistMessage(threadId, 'assistant', JSON.stringify({
+                type: 'tool-invocation',
+                toolCallId: msg.tool_call_id,
+                toolName: tc.toolName,
+                state: 'output-available',
+                input: completeArgs,
+                output: msg.content,
+              }), options);
+              pendingToolCalls.delete(msg.tool_call_id);
+            }
+          }
+          // Skip other message types (human, system)
 
-        // Collect conversation parts during streaming, add to memory in one batch at the end
-        let resultSummary = '';
-        const memoryParts = [];
-        const headlessPendingToolCalls = new Map();
-        let pendingText = '';
+        } else {
+          // ── Side channel: headless container chunks ───────────────────────
+          let chunk = item;
 
-        let lastEmittedText = '';
-        for await (const chunk of parseHeadlessStream(logStream, headlessContainer.codingAgent)) {
           // Result summary: skip if duplicate, otherwise ensure it starts on a new line
           if (chunk._resultSummary && chunk.type === 'text') {
             resultSummary = chunk._resultSummary;
@@ -317,11 +331,12 @@ async function* chatStream(threadId, message, attachments = [], options = {}) {
             }
             chunk = { ...chunk, text: '\n\n' + chunk.text };
           }
-          yield chunk;
+
           if (chunk.type === 'text') {
             fullText += chunk.text;
             memoryParts.push(chunk.text);
             pendingText += chunk.text;
+            yield chunk;
           } else if (chunk.type === 'tool-call') {
             // Flush accumulated text before tool call
             if (pendingText) {
@@ -330,67 +345,53 @@ async function* chatStream(threadId, message, attachments = [], options = {}) {
             }
             memoryParts.push('[tool-call] ' + chunk.toolName + ': ' + JSON.stringify(chunk.args));
             headlessPendingToolCalls.set(chunk.toolCallId, { toolName: chunk.toolName, args: chunk.args });
+            yield chunk;
           } else if (chunk.type === 'tool-result') {
+            // Enrich with args from matching tool-call (required by api.js tool-input-available update)
+            const htc = headlessPendingToolCalls.get(chunk.toolCallId);
+            const enriched = htc ? { ...chunk, args: htc.args, toolName: htc.toolName } : chunk;
+            yield enriched;
             memoryParts.push('[tool-result] ' + chunk.result);
-            const tc = headlessPendingToolCalls.get(chunk.toolCallId);
-            if (tc) {
+            if (htc) {
               persistMessage(threadId, 'assistant', JSON.stringify({
                 type: 'tool-invocation',
                 toolCallId: chunk.toolCallId,
-                toolName: tc.toolName,
+                toolName: htc.toolName,
                 state: 'output-available',
-                input: tc.args,
+                input: htc.args,
                 output: chunk.result,
               }), options);
               headlessPendingToolCalls.delete(chunk.toolCallId);
             }
+          } else {
+            // unknown events pass through unchanged
+            yield chunk;
           }
-          if (chunk._resultSummary) resultSummary = chunk._resultSummary;
-        }
 
-        // Flush remaining accumulated text
-        if (pendingText) {
-          persistMessage(threadId, 'assistant', pendingText, options);
-          pendingText = '';
+          if (chunk._resultSummary) resultSummary = chunk._resultSummary;
         }
+      }
+    } finally {
+      // Ensure no dangling promise when tool was never called
+      sideChannel.done();
+    }
 
-        // Container has exited by now (tailContainerLogs follows until EOF)
-        const exitCode = await waitForContainer(headlessContainer.containerName);
-        await removeContainer(headlessContainer.containerName);
-
-        if (exitCode === 0) {
-          const completionMsg = codeModeType === 'plan'
-            ? '\n\nPlanning complete.'
-            : '\n\nCoding complete.';
-          yield { type: 'text', text: completionMsg };
-          fullText += completionMsg;
-          persistMessage(threadId, 'assistant', completionMsg, options);
-        } else {
-          const failureMsg = '\n\nTask exited with errors.';
-          yield { type: 'text', text: failureMsg };
-          fullText += failureMsg;
-          persistMessage(threadId, 'assistant', failureMsg, options);
-        }
+    // Flush remaining channel text
+    if (pendingText) {
+      persistMessage(threadId, 'assistant', pendingText, options);
+    }
 
-        // Inject full conversation into LangGraph memory using the correct agent
-        if (memoryParts.length > 0) {
-          await agent.updateState(
-            { configurable: { thread_id: threadId } },
-            { messages: [new AIMessage(memoryParts.join('\n'))] }
-          );
-        }
-        // Also inject the summary separately for concise follow-up context
-        if (resultSummary) {
-          await agent.updateState(
-            { configurable: { thread_id: threadId } },
-            { messages: [new AIMessage(resultSummary)] }
-          );
-        }
+    // Persist LLM text (direct response with no tool, or LLM follow-up after container)
+    if (llmTextAccum) {
+      persistMessage(threadId, 'assistant', llmTextAccum, options);
+    }
 
-      } catch (err) {
-        console.error('[chatStream] headless stream error:', err);
-        yield { type: 'text', text: '\n\nError streaming headless output: ' + err.message };
-      }
+    // Inject full headless conversation detail into LangGraph memory for follow-up turns
+    if (memoryParts.length > 0) {
+      await agent.updateState(
+        { configurable: { thread_id: threadId } },
+        { messages: [new AIMessage(memoryParts.join('\n'))] }
+      );
     }
 
   } catch (err) {