thepopebot 1.2.75-beta.1 → 1.2.75-beta.12

package/README.md

@@ -106,7 +106,7 @@ The wizard walks you through everything:
 - **Web Chat**: Visit your APP_URL to chat with your agent, create jobs, upload files
 - **Telegram** (optional): Run `npm run setup-telegram` to connect a Telegram bot
 - **Webhook**: Send a POST to `/api/create-agent-job` with your API key to create jobs programmatically
-- **Cron**: Edit `config/CRONS.json` to schedule recurring jobs
+- **Cron**: Edit `agent-job/CRONS.json` to schedule recurring jobs
 
 ### Chat vs Agent LLM
 
@@ -192,6 +192,17 @@ See [Different Models](docs/RUNNING_DIFFERENT_MODELS.md) for the full provider r
 
 ---
 
+## Known Issues
+
+### Windows: `SQLITE_IOERR_SHMOPEN`
+
+SQLite can't create or open its shared-memory (`.shm`) file. Common causes:
+
+- **Antivirus** (Windows Defender, etc.) locking the database files — add your project folder to the exclusion list
+- **Cloud-synced folders** (OneDrive, Dropbox, Google Drive) — move your project to a non-synced directory like `C:\Projects\`
+
+---
+
 ## Docs
 
 | Document | Description |

package/bin/CLAUDE.md

@@ -21,7 +21,7 @@ Entry point: `cli.js` (invoked via `npx thepopebot <command>`).
 
 `managed-paths.js` defines files auto-synced by `init`. These are overwritten on every init/upgrade — users should not edit them.
 
-**Managed paths**: `.github/workflows/`, `docker-compose.yml`, `.dockerignore`, `.gitignore`, `CLAUDE.md`, `config/CLAUDE.md`, `skills/CLAUDE.md`, `cron/CLAUDE.md`, `triggers/CLAUDE.md`, `docs/CLAUDE.md`.
+**Managed paths**: `.github/workflows/`, `docker-compose.yml`, `.dockerignore`, `.gitignore`, `agent-job/CLAUDE.md`, `event-handler/CLAUDE.md`, `skills/CLAUDE.md`.
 
 `isManaged(relPath)` — returns true if a path is managed (exact match or directory prefix).
 

package/bin/cli.js

@@ -52,11 +52,13 @@ Commands:
   init                              Scaffold a new thepopebot project
   upgrade|update [@beta|version]    Upgrade thepopebot (install, init, build, commit, push)
   setup                             Run interactive setup wizard
+  setup-ssl                         Configure SSL with Let's Encrypt wildcard cert
   setup-telegram                    Reconfigure Telegram webhook
   reset-auth                        Regenerate AUTH_SECRET (invalidates all sessions)
   reset [file]                      Restore a template file (or list available templates)
   diff [file]                       Show differences between project files and package templates
   sync <path>                       Sync local package to a test install (build, pack, Docker)
+  sync --fast <path>                Fast sync — copy source into running container, rebuild .next
   set-var <KEY> [VALUE]             Set a GitHub repository variable
   user:password <email>             Change a user's password
 `);
@@ -246,6 +248,7 @@ async function init() {
     const pkg = {
       name: dirName,
       private: true,
+      type: 'module',
       scripts: {
         setup: 'thepopebot setup',
         'setup-telegram': 'thepopebot setup-telegram',
@@ -258,11 +261,19 @@ async function init() {
     fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n');
     console.log('  Created package.json');
   } else {
-    console.log('  Skipped package.json (already exists)');
+    // Ensure "type": "module" is set for ESM support
+    const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+    if (!pkg.type) {
+      pkg.type = 'module';
+      fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n');
+      console.log('  Added "type": "module" to package.json');
+    } else {
+      console.log('  Skipped package.json (already exists)');
+    }
   }
 
   // Create default skill activation symlinks
-  const defaultSkills = ['agent-job-secrets'];
+  const defaultSkills = [];
   const activeDir = path.join(cwd, 'skills', 'active');
   fs.mkdirSync(activeDir, { recursive: true });
   for (const skill of defaultSkills) {
@@ -337,8 +348,7 @@ AUTH_TRUST_HOST=true
 DATABASE_PATH=data/db/thepopebot.sqlite
 THEPOPEBOT_VERSION=${version}
 
-# Uncomment to use a custom docker-compose file that won't be overwritten by upgrades.
-# Edit docker-compose.custom.yml with your changes, then uncomment:
+# To enable SSL with Let's Encrypt, run: npx thepopebot setup-ssl
 # COMPOSE_FILE=docker-compose.custom.yml
 `;
     fs.writeFileSync(envPath, seedEnv);
@@ -375,7 +385,7 @@ function reset(filePath) {
       console.log(`  ${destPath(file)}`);
     }
     console.log('\nUsage: thepopebot reset <file>');
-    console.log('Example: thepopebot reset config/SOUL.md\n');
+    console.log('Example: thepopebot reset agent-job/SOUL.md\n');
     return;
   }
 
@@ -432,7 +442,7 @@ function diff(filePath) {
       console.log('  All files match package templates.');
     }
     console.log('\nUsage: thepopebot diff <file>');
-    console.log('Example: thepopebot diff config/SOUL.md\n');
+    console.log('Example: thepopebot diff agent-job/SOUL.md\n');
     return;
   }
 
@@ -490,6 +500,15 @@ function setup() {
   }
 }
 
+function setupSsl() {
+  const setupScript = path.join(__dirname, '..', 'setup', 'setup-ssl.mjs');
+  try {
+    execFileSync(process.execPath, [setupScript], { stdio: 'inherit', cwd: process.cwd() });
+  } catch {
+    process.exit(1);
+  }
+}
+
 function setupTelegram() {
   const setupScript = path.join(__dirname, '..', 'setup', 'setup-telegram.mjs');
   try {
@@ -770,6 +789,9 @@ switch (command) {
   case 'setup':
     setup();
     break;
+  case 'setup-ssl':
+    setupSsl();
+    break;
   case 'setup-telegram':
     setupTelegram();
     break;
@@ -787,8 +809,15 @@ switch (command) {
     await upgrade();
     break;
   case 'sync': {
-    const { sync } = await import('./sync.js');
-    await sync(args[0]);
+    const fast = args.includes('--fast');
+    const syncArgs = args.filter(a => a !== '--fast');
+    if (fast) {
+      const { syncFast } = await import('./sync.js');
+      await syncFast(syncArgs[0]);
+    } else {
+      const { sync } = await import('./sync.js');
+      await sync(syncArgs[0]);
+    }
     break;
   }
   case 'set-var':

package/bin/docker-build.js

@@ -62,6 +62,11 @@ const CODING_AGENTS = [
     context: 'docker/coding-agent',
     dockerfile: 'docker/coding-agent/Dockerfile.opencode',
   },
+  {
+    name: 'coding-agent-kimi-cli',
+    context: 'docker/coding-agent',
+    dockerfile: 'docker/coding-agent/Dockerfile.kimi-cli',
+  },
 ];
 
 // Non-coding-agent images (independent, built in parallel)

package/bin/managed-paths.js

@@ -8,12 +8,9 @@ export const MANAGED_PATHS = [
   'docker-compose.yml',
   '.dockerignore',
   '.gitignore',
-  'CLAUDE.md',
-  'config/CLAUDE.md',
+  'agent-job/CLAUDE.md',
+  'event-handler/CLAUDE.md',
   'skills/CLAUDE.md',
-  'cron/CLAUDE.md',
-  'triggers/CLAUDE.md',
-  'docs/',
 ];
 
 export function isManaged(relPath) {

package/bin/sync.js

@@ -306,6 +306,90 @@ function buildDockerImage(projectPath) {
   }
 }
 
+/**
+ * Fast sync — skip Docker image rebuild entirely.
+ *
+ *   1. Build package JSX (npm run build)
+ *   2. mirrorTemplates() — scaffold using init's managed-path logic
+ *   3. docker cp package source (lib/, api/, config/, package.json) into
+ *      the running container's /app/node_modules/thepopebot/
+ *   4. docker cp web/app/ + web/postcss.config.mjs into container
+ *   5. docker exec next build inside the container (tailwindcss already there)
+ *   6. Clean up copied source from container
+ *   7. docker exec pm2 restart all
+ */
+export async function syncFast(projectPath) {
+  if (!projectPath) {
+    console.error('\n  Usage: thepopebot sync --fast <path-to-project>\n');
+    process.exit(1);
+  }
+
+  projectPath = path.resolve(projectPath);
+
+  if (!fs.existsSync(path.join(projectPath, 'package.json'))) {
+    console.error(`\n  Not a project directory (no package.json): ${projectPath}\n`);
+    process.exit(1);
+  }
+
+  // 1. Build JSX
+  console.log('\n  Building package...');
+  execSync('npm run build', { stdio: 'inherit', cwd: PACKAGE_DIR });
+
+  // 2. Mirror templates
+  console.log('\n  Mirroring templates...');
+  mirrorTemplates(projectPath);
+
+  // 3. Get running container ID
+  const container = execSync('docker compose ps -q event-handler', {
+    encoding: 'utf8',
+    cwd: projectPath,
+  }).trim();
+
+  if (!container) {
+    console.error('\n  event-handler container is not running. Use full sync instead.\n');
+    process.exit(1);
+  }
+
+  // 4. Copy package source into container's node_modules/thepopebot/
+  const PKG_DEST = '/app/node_modules/thepopebot';
+  const PACKAGE_DIRS = ['lib', 'api', 'config'];
+
+  console.log('\n  Copying package source into container...');
+  for (const dir of PACKAGE_DIRS) {
+    execSync(`docker exec ${container} rm -rf ${PKG_DEST}/${dir}`, { stdio: 'inherit' });
+    execSync(`docker cp ${path.join(PACKAGE_DIR, dir)} ${container}:${PKG_DEST}/${dir}`, { stdio: 'inherit' });
+  }
+  // Also copy package.json for exports resolution
+  execSync(`docker cp ${path.join(PACKAGE_DIR, 'package.json')} ${container}:${PKG_DEST}/package.json`, { stdio: 'inherit' });
+
+  // 5. Copy web/app/ source into container for next build
+  const webDir = path.join(PACKAGE_DIR, 'web');
+  console.log('\n  Copying web source into container...');
+  execSync(`docker cp ${path.join(webDir, 'app')} ${container}:/app/app`, { stdio: 'inherit' });
+  execSync(`docker cp ${path.join(webDir, 'postcss.config.mjs')} ${container}:/app/postcss.config.mjs`, { stdio: 'inherit' });
+  execSync(`docker cp ${path.join(webDir, 'next.config.mjs')} ${container}:/app/next.config.mjs`, { stdio: 'inherit' });
+
+  // 6. Run next build inside the container
+  // Hide data/logs dirs so webpack's FileSystemInfo doesn't crawl them (causes OOM/RangeError
+  // when workspaces contain thousands of files). Restored immediately after build.
+  console.log('\n  Building Next.js inside container...');
+  execSync(`docker exec ${container} sh -c 'mv /app/data /app/.data-build-tmp 2>/dev/null; mv /app/logs /app/.logs-build-tmp 2>/dev/null; true'`, { stdio: 'inherit' });
+  try {
+    execSync(`docker exec ${container} ./node_modules/.bin/next build`, { stdio: 'inherit' });
+  } finally {
+    execSync(`docker exec ${container} sh -c 'mv /app/.data-build-tmp /app/data 2>/dev/null; mv /app/.logs-build-tmp /app/logs 2>/dev/null; true'`, { stdio: 'inherit' });
+  }
+
+  // 7. Clean up web source from container (not needed at runtime)
+  execSync(`docker exec ${container} rm -rf /app/app`, { stdio: 'inherit' });
+
+  // 8. Restart PM2
+  console.log('\n  Restarting server...');
+  execSync(`docker exec ${container} pm2 restart all`, { stdio: 'inherit' });
+
+  console.log('\n  Fast synced!\n');
+}
+
 export async function sync(projectPath) {
   if (!projectPath) {
     console.error('\n  Usage: thepopebot sync <path-to-project>\n');

package/config/CLAUDE.md

@@ -1,32 +1,4 @@
-# config/ — Configuration Files
-
-## Directory Structure
-
-```
-config/
-├── agent-chat/
-│   └── SYSTEM.md              # Agent chat system prompt (supports {{skills}}, {{datetime}})
-├── code-chat/
-│   └── SYSTEM.md              # Code workspace system prompt
-├── agent-job/
-│   ├── SOUL.md                # Agent personality/identity (used by Docker agent)
-│   ├── AGENT_JOB.md           # Agent runtime environment docs (used by Docker agent)
-│   └── SUMMARY.md             # Prompt for summarizing completed jobs
-├── cluster/
-│   ├── SYSTEM.md              # Cluster worker system prompt
-│   └── ROLE.md                # Per-role prompt template for cluster workers
-├── HEARTBEAT.md               # Self-monitoring behavior (cron task prompt)
-├── CRONS.json                 # Scheduled job definitions
-└── TRIGGERS.json              # Webhook trigger definitions
-```
-
-## Markdown File Includes
-
-Markdown files in `config/` support includes and built-in variables, powered by `lib/utils/render-md.js`.
-
-- **File includes**: `{{ filepath.md }}` — resolves relative to project root, recursive with circular detection. Missing files are left as-is.
-- **`{{datetime}}`** — Current ISO timestamp.
-- **`{{skills}}`** — Dynamic bullet list of active skill descriptions from `skills/active/*/SKILL.md` frontmatter. Never hardcode skill names — this is resolved at runtime.
+# config/ — Next.js Config Wrapper
 
 ## Next.js Config Wrapper (index.js)
 

package/config/instrumentation.js

@@ -70,7 +70,7 @@ export async function register() {
   const { startClusterRuntime } = await import('../lib/cluster/runtime.js');
   startClusterRuntime();
 
-  // Start internal maintenance cron (cleanup expired agent job keys, etc.)
+  // Start internal maintenance cron (cleanup orphaned agent job keys, etc.)
   const { startMaintenanceCron } = await import('../lib/maintenance.js');
   startMaintenanceCron();
 

package/lib/CLAUDE.md

@@ -12,16 +12,16 @@ If the task needs to *think*, use `agent`. If it just needs to *do*, use `comman
 
 **Agent**: Creates a Docker Agent job via `createAgentJob()`. Pushes an `agent-job/*` branch and launches a local Docker container. The `job` string is the LLM task prompt. Agent backend selected via `agent_backend` in `agent-job.config.json`.
 
-**Command**: Runs a shell command on the event handler. Working directory: `cron/` for crons, `triggers/` for triggers.
+**Command**: Runs a shell command on the event handler. Working directory: project root.
 
 **Webhook**: Makes an HTTP request. `GET` skips the body; `POST` (default) sends `{ ...vars }` or `{ ...vars, data: <payload> }`.
 
 ## Cron Jobs
 
-Defined in `config/CRONS.json`, loaded by `lib/cron.js` at startup via `node-cron`. Each entry has `name`, `schedule` (cron expression), `type` (`agent`/`command`/`webhook`), and the corresponding action fields (`job`, `command`, or `url`/`method`/`headers`/`vars`). Set `enabled: false` to disable. Agent-type entries support optional `llm_provider` and `llm_model` fields to override the default LLM (passed to Docker agent via `agent-job.config.json`).
+Defined in `agent-job/CRONS.json`, loaded by `lib/cron.js` at startup via `node-cron`. Each entry has `name`, `schedule` (cron expression), `type` (`agent`/`command`/`webhook`), and the corresponding action fields (`job`, `command`, or `url`/`method`/`headers`/`vars`). Set `enabled: false` to disable. Agent-type entries support optional `llm_provider` and `llm_model` fields to override the default LLM (passed to Docker agent via `agent-job.config.json`).
 
 ## Webhook Triggers
 
-Defined in `config/TRIGGERS.json`, loaded by `lib/triggers.js`. Each trigger watches an endpoint path (`watch_path`) and fires an array of actions (fire-and-forget, after auth, before route handler). Actions use the same `type`/`job`/`command`/`url` fields as cron jobs, including optional `llm_provider`/`llm_model` overrides.
+Defined in `event-handler/TRIGGERS.json`, loaded by `lib/triggers.js`. Each trigger watches an endpoint path (`watch_path`) and fires an array of actions (fire-and-forget, after auth, before route handler). Actions use the same `type`/`job`/`command`/`url` fields as cron jobs, including optional `llm_provider`/`llm_model` overrides.
 
 Template tokens in `job` and `command` strings: `{{body}}`, `{{body.field}}`, `{{query}}`, `{{query.field}}`, `{{headers}}`, `{{headers.field}}`.

package/lib/ai/CLAUDE.md

@@ -5,12 +5,12 @@
 Two agent singletons, both using `createReactAgent` from `@langchain/langgraph/prebuilt` with `SqliteSaver` for conversation memory:
 
 **Agent Chat** — singleton via `getAgentChat()`:
-- System prompt: `config/agent-chat/SYSTEM.md` (rendered fresh each invocation via `render_md()`)
+- System prompt: `event-handler/agent-chat/SYSTEM.md` (rendered fresh each invocation via `render_md()`)
 - Tools: `agent_job`, `coding_agent`
 - Call `resetAgentChats()` to clear both singletons (required if hot-reloading)
 
 **Code Chat** — singleton via `getCodeChat()`:
-- System prompt: `config/code-chat/SYSTEM.md` (rendered fresh each invocation)
+- System prompt: `event-handler/code-chat/SYSTEM.md` (rendered fresh each invocation)
 - Tools: `coding_agent` (reads repo/branch/workspace from `runtime.configurable`)
 
 ## Adding a New Tool
@@ -73,7 +73,28 @@ Three-layer parser for Claude Code agents running in headless Docker containers:
 3. **Event mapper** (`mapLine()`) — Converts each line to chat events:
    - `assistant` messages: `text` blocks → `{ type: 'text' }`, `tool_use` blocks → `{ type: 'tool-call' }`
    - `user` messages: `tool_result` blocks → `{ type: 'tool-result' }` (priority: stdout > string content > array)
-   - `result` messages: → `{ type: 'text', _resultSummary }` (injected into LangGraph memory)
+   - `result` messages: → `{ type: 'text' }` (final summary from the agent)
    - Non-JSON lines (e.g. `NO_CHANGES`, `AGENT_FAILED`): wrapped as plain text events
 
 `parseHeadlessStream(dockerLogStream)` is an async generator consuming `http.IncomingMessage`. `mapLine()` is also reused by `lib/cluster/stream.js` for worker log parsing.
+
+### Tool Return Format
+
+The `coding_agent` tool (in `tools.js`) returns the **full container session** as a flat JSON array. This becomes the ToolMessage in LangGraph's checkpoint, giving the LLM complete context on the current turn. The array contains:
+
+- `{ type: 'meta', codingAgent, backendApi }` — first event, agent identity
+- `{ type: 'text', text }` — agent text output
+- `{ type: 'tool-call', toolCallId, toolName, args }` — agent tool invocations
+- `{ type: 'tool-result', toolCallId, result }` — tool execution results
+- `{ type: 'exit', exitCode }` — last event, container exit status
+
+On error before streaming starts: `[{ type: 'error', message }]`.
+
+### Adding a New Agent Mapper (line-mappers.js)
+
+Each coding agent CLI has its own mapper function (`mapClaudeCodeLine`, `mapPiLine`, `mapGeminiLine`, `mapCodexLine`, `mapOpenCodeLine`, `mapKimiLine`). When adding a new agent:
+
+1. Create `mapXxxLine(parsed)` in `line-mappers.js` that returns an array of `{ type, ... }` events
+2. Register it in `headless-stream.js`: add to imports, re-exports, and the `mapperMap` object
+3. Map the agent's JSON output to three event types: `{ type: 'text', text }`, `{ type: 'tool-call', toolCallId, toolName, args }`, `{ type: 'tool-result', toolCallId, result }`
+4. Return `[{ type: 'skip' }]` for noise events (session init, rate limits, etc.) to suppress them without triggering the unknown fallback

package/lib/ai/agent.js

@@ -3,7 +3,8 @@ import { SystemMessage } from '@langchain/core/messages';
 import { createModel } from './model.js';
 import { agentJobTool, agentChatCodingTool, codeChatCodingTool } from './tools.js';
 import { SqliteSaver } from '@langchain/langgraph-checkpoint-sqlite';
-import { agentJobPlanningMd, codePlanningMd, thepopebotDb } from '../paths.js';
+import path from 'path';
+import { PROJECT_ROOT } from '../paths.js';
 import { render_md } from '../utils/render-md.js';
 
 // Singletons on globalThis to survive Next.js webpack chunk duplication.
@@ -20,13 +21,14 @@ export async function getAgentChat() {
     const model = await createModel();
     const tools = [agentJobTool, agentChatCodingTool];
 
-    const checkpointer = SqliteSaver.fromConnString(thepopebotDb);
+    const dbPath = process.env.DATABASE_PATH || path.join(PROJECT_ROOT, 'data/db/thepopebot.sqlite');
+    const checkpointer = SqliteSaver.fromConnString(dbPath);
 
     globalThis.__popebotAgentChat = createReactAgent({
       llm: model,
       tools,
       checkpointSaver: checkpointer,
-      prompt: (state) => [new SystemMessage(render_md(agentJobPlanningMd)), ...state.messages],
+      prompt: (state) => [new SystemMessage(render_md(path.join(PROJECT_ROOT, 'event-handler/agent-chat/SYSTEM.md'))), ...state.messages],
     });
   }
   return globalThis.__popebotAgentChat;
@@ -41,13 +43,14 @@ export async function getCodeChat() {
     const model = await createModel();
     const tools = [codeChatCodingTool];
 
-    const checkpointer = SqliteSaver.fromConnString(thepopebotDb);
+    const dbPath = process.env.DATABASE_PATH || path.join(PROJECT_ROOT, 'data/db/thepopebot.sqlite');
+    const checkpointer = SqliteSaver.fromConnString(dbPath);
 
     globalThis.__popebotCodeChat = createReactAgent({
       llm: model,
       tools,
       checkpointSaver: checkpointer,
-      prompt: (state) => [new SystemMessage(render_md(codePlanningMd)), ...state.messages],
+      prompt: (state) => [new SystemMessage(render_md(path.join(PROJECT_ROOT, 'event-handler/code-chat/SYSTEM.md'))), ...state.messages],
     });
   }
   return globalThis.__popebotCodeChat;

package/lib/ai/async-channel.js

@@ -0,0 +1,51 @@
+/**
+ * Async push/pull queue. Producer calls push()/done(), consumer uses for-await.
+ */
+export function createChannel() {
+  const queue = [];
+  const waiters = [];
+  let isDone = false;
+
+  return {
+    push(value) {
+      if (waiters.length > 0) waiters.shift()(value);
+      else queue.push(value);
+    },
+    done() {
+      isDone = true;
+      while (waiters.length > 0) waiters.shift()(Symbol.for('done'));
+    },
+    async *[Symbol.asyncIterator]() {
+      while (true) {
+        if (queue.length > 0) {
+          yield queue.shift();
+        } else if (isDone) {
+          return;
+        } else {
+          const value = await new Promise(resolve => waiters.push(resolve));
+          if (value === Symbol.for('done')) return;
+          yield value;
+        }
+      }
+    }
+  };
+}
+
+/**
+ * Merge two async iterables — yields from whichever has data first.
+ * Completes when BOTH are exhausted.
+ */
+export async function* mergeAsyncIterables(iter1, iter2) {
+  const channel = createChannel();
+  let active = 2;
+
+  const consume = async (iter) => {
+    for await (const item of iter) channel.push(item);
+    if (--active === 0) channel.done();
+  };
+
+  consume(iter1);
+  consume(iter2);
+
+  yield* channel;
+}

package/lib/ai/headless-stream.js

@@ -10,6 +10,7 @@ export {
   mapGeminiLine,
   mapCodexLine,
   mapOpenCodeLine,
+  mapKimiLine,
 } from './line-mappers.js';
 
 import {
@@ -19,6 +20,7 @@ import {
   mapGeminiLine,
   mapCodexLine,
   mapOpenCodeLine,
+  mapKimiLine,
 } from './line-mappers.js';
 
 /**
@@ -41,6 +43,7 @@ export async function* parseHeadlessStream(dockerLogStream, codingAgent = 'claud
     'gemini-cli': mapGeminiLine,
     'codex-cli': mapCodexLine,
     'opencode': mapOpenCodeLine,
+    'kimi-cli': mapKimiLine,
   };
   const mapper = mapperMap[codingAgent] || mapClaudeCodeLine;