thepopebot 1.2.74 → 1.2.75-beta.10

package/README.md

@@ -192,6 +192,17 @@ See [Different Models](docs/RUNNING_DIFFERENT_MODELS.md) for the full provider r
 
 ---
 
+## Known Issues
+
+### Windows: `SQLITE_IOERR_SHMOPEN`
+
+SQLite can't create or open its shared-memory (`.shm`) file. Common causes:
+
+- **Antivirus** (Windows Defender, etc.) locking the database files — add your project folder to the exclusion list
+- **Cloud-synced folders** (OneDrive, Dropbox, Google Drive) — move your project to a non-synced directory like `C:\Projects\`
+
+---
+
 ## Docs
 
 | Document | Description |

package/api/CLAUDE.md

@@ -25,6 +25,8 @@ Browser-facing data fetching uses **fetch route handlers** colocated with pages
 |--------|------|------|---------|
 | GET | `/api/ping` | None | Health check |
 | POST | `/api/create-agent-job` | `x-api-key` | Create agent job |
+| GET | `/api/get-agent-job-secret` | `x-api-key` | Get an agent job secret; oauth2 credentials return only the access_token (auto-refreshed) |
+| POST | `/api/set-agent-job-secret` | `x-api-key` | Set/update an agent job secret (for agents to persist rotated credentials) |
 | GET | `/api/agent-jobs/status` | `x-api-key` | Agent job status (query: `?agent_job_id=`) |
 | POST | `/api/telegram/webhook` | Telegram webhook secret | Telegram message handler |
 | POST | `/api/telegram/register` | `x-api-key` | Register bot token + webhook URL |

package/api/index.js

@@ -11,6 +11,9 @@ import { getConfig } from '../lib/config.js';
 import { parseOAuthState, exchangeCodeForToken } from '../lib/oauth/helper.js';
 import { setAgentJobSecret } from '../lib/db/config.js';
 
+// ── Per-key lock for OAuth token refresh ────────────────────────────
+const _refreshLocks = new Map();
+
 // Bot token — resolved from DB/env, can be overridden by /telegram/register
 let telegramBotToken = null;
 
@@ -104,6 +107,81 @@ async function handleCreateAgentJob(request) {
   }
 }
 
+async function handleGetAgentSecret(request) {
+  const record = verifyApiKey(request.headers.get('x-api-key'));
+  if (record.type !== 'agent_job_api_key') {
+    return Response.json({ error: 'Forbidden' }, { status: 403 });
+  }
+
+  const key = new URL(request.url).searchParams.get('key');
+  if (!key) return Response.json({ error: 'Missing key' }, { status: 400 });
+
+  const { getAgentJobSecretRaw, setAgentJobSecret: saveSecret } = await import('../lib/db/config.js');
+  const raw = getAgentJobSecretRaw(key);
+  if (!raw) return Response.json({ error: 'Not found' }, { status: 404 });
+
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    // Plain string
+    return Response.json({ value: raw });
+  }
+
+  if (parsed.type === 'oauth2') {
+    // Serialize refresh per key — prevents concurrent requests from racing on token rotation
+    if (!_refreshLocks.has(key)) _refreshLocks.set(key, Promise.resolve());
+    let release;
+    const gate = new Promise((r) => { release = r; });
+    const prev = _refreshLocks.get(key);
+    _refreshLocks.set(key, gate);
+    await prev;
+
+    try {
+      // Re-read after acquiring lock — previous request may have already refreshed
+      const freshRaw = getAgentJobSecretRaw(key);
+      const freshParsed = freshRaw ? JSON.parse(freshRaw) : parsed;
+
+      const { refreshOAuthToken } = await import('../lib/oauth/helper.js');
+      const newToken = await refreshOAuthToken({
+        refreshToken: freshParsed.token.refresh_token,
+        clientId: freshParsed.clientId,
+        clientSecret: freshParsed.clientSecret,
+        tokenUrl: freshParsed.tokenUrl,
+      });
+      // Persist updated token (refresh token may have rotated)
+      saveSecret(key, JSON.stringify({ ...freshParsed, token: { ...freshParsed.token, ...newToken } }), 'refresh');
+      return Response.json({ value: newToken.access_token });
+    } catch (err) {
+      console.error(`[secrets] OAuth refresh failed for "${key}":`, err.message);
+      return Response.json({ error: `OAuth refresh failed: ${err.message}` }, { status: 502 });
+    } finally {
+      release();
+    }
+  }
+  if (parsed.type === 'oauth_token') {
+    return Response.json({ value: JSON.stringify(parsed.token) });
+  }
+  // Unknown structured value — return raw
+  return Response.json({ value: raw });
+}
+
+async function handleSetAgentSecret(request) {
+  const record = verifyApiKey(request.headers.get('x-api-key'));
+  if (record.type !== 'agent_job_api_key') {
+    return Response.json({ error: 'Forbidden' }, { status: 403 });
+  }
+
+  const body = await request.json();
+  const { key, value } = body;
+  if (!key || typeof value !== 'string') {
+    return Response.json({ error: 'Missing key or value' }, { status: 400 });
+  }
+  const { setAgentJobSecret } = await import('../lib/db/config.js');
+  setAgentJobSecret(key, value, 'agent');
+  return Response.json({ success: true });
+}
+
 async function handleTelegramRegister(request) {
   const body = await request.json();
   const { bot_token, webhook_url } = body;
@@ -248,9 +326,21 @@ async function handleOAuthCallback(request) {
       redirectUri,
     });
 
-    // Save the full token JSON as the secret value
-    const tokenJson = JSON.stringify(tokenData);
-    setAgentJobSecret(state.secretName, tokenJson, 'oauth');
+    // Save token with typed wrapper so the API can auto-refresh on fetch
+    const secretType = state.secretType || 'oauth2';
+    let stored;
+    if (secretType === 'oauth_token') {
+      stored = JSON.stringify({ type: 'oauth_token', token: tokenData });
+    } else {
+      stored = JSON.stringify({
+        type: 'oauth2',
+        token: tokenData,
+        clientId: state.clientId,
+        clientSecret: state.clientSecret,
+        tokenUrl: state.tokenUrl,
+      });
+    }
+    setAgentJobSecret(state.secretName, stored, 'oauth');
 
     return oauthResultPage(true, state.secretName);
   } catch (err) {
@@ -315,6 +405,7 @@ async function POST(request) {
   // Route to handler
   switch (routePath) {
     case '/create-agent-job':     return handleCreateAgentJob(request);
+    case '/set-agent-job-secret':  return handleSetAgentSecret(request);
     case '/telegram/webhook':   return handleTelegramWebhook(request);
     case '/telegram/register':  return handleTelegramRegister(request);
     case '/github/webhook':     return handleGithubWebhook(request);
@@ -333,6 +424,7 @@ async function GET(request) {
   switch (routePath) {
     case '/ping':               return Response.json({ message: 'Pong!' });
     case '/agent-jobs/status':  return handleAgentJobStatus(request);
+    case '/get-agent-job-secret':   return handleGetAgentSecret(request);
     case '/oauth/callback':     return handleOAuthCallback(request);
     default:                    return Response.json({ error: 'Not found' }, { status: 404 });
   }

package/bin/cli.js

@@ -52,11 +52,13 @@ Commands:
   init                              Scaffold a new thepopebot project
   upgrade|update [@beta|version]    Upgrade thepopebot (install, init, build, commit, push)
   setup                             Run interactive setup wizard
+  setup-ssl                         Configure SSL with Let's Encrypt wildcard cert
   setup-telegram                    Reconfigure Telegram webhook
   reset-auth                        Regenerate AUTH_SECRET (invalidates all sessions)
   reset [file]                      Restore a template file (or list available templates)
   diff [file]                       Show differences between project files and package templates
   sync <path>                       Sync local package to a test install (build, pack, Docker)
+  sync --fast <path>                Fast sync — copy source into running container, rebuild .next
   set-var <KEY> [VALUE]             Set a GitHub repository variable
   user:password <email>             Change a user's password
 `);
@@ -246,6 +248,7 @@ async function init() {
     const pkg = {
       name: dirName,
       private: true,
+      type: 'module',
       scripts: {
         setup: 'thepopebot setup',
         'setup-telegram': 'thepopebot setup-telegram',
@@ -258,11 +261,19 @@ async function init() {
     fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n');
     console.log('  Created package.json');
   } else {
-    console.log('  Skipped package.json (already exists)');
+    // Ensure "type": "module" is set for ESM support
+    const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+    if (!pkg.type) {
+      pkg.type = 'module';
+      fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n');
+      console.log('  Added "type": "module" to package.json');
+    } else {
+      console.log('  Skipped package.json (already exists)');
+    }
   }
 
   // Create default skill activation symlinks
-  const defaultSkills = ['get-secret'];
+  const defaultSkills = [];
   const activeDir = path.join(cwd, 'skills', 'active');
   fs.mkdirSync(activeDir, { recursive: true });
   for (const skill of defaultSkills) {
@@ -337,8 +348,7 @@ AUTH_TRUST_HOST=true
 DATABASE_PATH=data/db/thepopebot.sqlite
 THEPOPEBOT_VERSION=${version}
 
-# Uncomment to use a custom docker-compose file that won't be overwritten by upgrades.
-# Edit docker-compose.custom.yml with your changes, then uncomment:
+# To enable SSL with Let's Encrypt, run: npx thepopebot setup-ssl
 # COMPOSE_FILE=docker-compose.custom.yml
 `;
     fs.writeFileSync(envPath, seedEnv);
@@ -490,6 +500,15 @@ function setup() {
   }
 }
 
+function setupSsl() {
+  const setupScript = path.join(__dirname, '..', 'setup', 'setup-ssl.mjs');
+  try {
+    execFileSync(process.execPath, [setupScript], { stdio: 'inherit', cwd: process.cwd() });
+  } catch {
+    process.exit(1);
+  }
+}
+
 function setupTelegram() {
   const setupScript = path.join(__dirname, '..', 'setup', 'setup-telegram.mjs');
   try {
@@ -770,6 +789,9 @@ switch (command) {
   case 'setup':
     setup();
     break;
+  case 'setup-ssl':
+    setupSsl();
+    break;
   case 'setup-telegram':
     setupTelegram();
     break;
@@ -787,8 +809,15 @@ switch (command) {
     await upgrade();
     break;
   case 'sync': {
-    const { sync } = await import('./sync.js');
-    await sync(args[0]);
+    const fast = args.includes('--fast');
+    const syncArgs = args.filter(a => a !== '--fast');
+    if (fast) {
+      const { syncFast } = await import('./sync.js');
+      await syncFast(syncArgs[0]);
+    } else {
+      const { sync } = await import('./sync.js');
+      await sync(syncArgs[0]);
+    }
     break;
   }
   case 'set-var':

package/bin/docker-build.js

@@ -62,6 +62,11 @@ const CODING_AGENTS = [
     context: 'docker/coding-agent',
     dockerfile: 'docker/coding-agent/Dockerfile.opencode',
   },
+  {
+    name: 'coding-agent-kimi-cli',
+    context: 'docker/coding-agent',
+    dockerfile: 'docker/coding-agent/Dockerfile.kimi-cli',
+  },
 ];
 
 // Non-coding-agent images (independent, built in parallel)

package/bin/managed-paths.js

@@ -13,7 +13,7 @@ export const MANAGED_PATHS = [
   'skills/CLAUDE.md',
   'cron/CLAUDE.md',
   'triggers/CLAUDE.md',
-  'docs/CLAUDE.md',
+  'docs/',
 ];
 
 export function isManaged(relPath) {

package/bin/sync.js

@@ -277,8 +277,7 @@ function buildDockerImage(projectPath) {
   fs.cpSync(webSrc, webDest, { recursive: true });
 
   try {
-    // Build using stdin Dockerfile with project dir as context (no cache to ensure fresh package)
-    execSync(`docker build --no-cache -f - -t ${imageTag} .`, {
+    execSync(`docker build -f - -t ${imageTag} .`, {
       input: dockerfile,
       stdio: ['pipe', 'inherit', 'inherit'],
       cwd: projectPath,
@@ -287,6 +286,12 @@ function buildDockerImage(projectPath) {
     fs.rmSync(webDest, { recursive: true, force: true });
   }
 
+  // Clean up dangling images from previous builds
+  try {
+    execSync('docker image prune -f', { stdio: 'ignore' });
+  } catch {}
+
+
   // Update THEPOPEBOT_VERSION in .env
   const envPath = path.join(projectPath, '.env');
   if (fs.existsSync(envPath)) {
@@ -301,6 +306,90 @@ function buildDockerImage(projectPath) {
   }
 }
 
+/**
+ * Fast sync — skip Docker image rebuild entirely.
+ *
+ *   1. Build package JSX (npm run build)
+ *   2. mirrorTemplates() — scaffold using init's managed-path logic
+ *   3. docker cp package source (lib/, api/, config/, package.json) into
+ *      the running container's /app/node_modules/thepopebot/
+ *   4. docker cp web/app/ + web/postcss.config.mjs into container
+ *   5. docker exec next build inside the container (tailwindcss already there)
+ *   6. Clean up copied source from container
+ *   7. docker exec pm2 restart all
+ */
+export async function syncFast(projectPath) {
+  if (!projectPath) {
+    console.error('\n  Usage: thepopebot sync --fast <path-to-project>\n');
+    process.exit(1);
+  }
+
+  projectPath = path.resolve(projectPath);
+
+  if (!fs.existsSync(path.join(projectPath, 'package.json'))) {
+    console.error(`\n  Not a project directory (no package.json): ${projectPath}\n`);
+    process.exit(1);
+  }
+
+  // 1. Build JSX
+  console.log('\n  Building package...');
+  execSync('npm run build', { stdio: 'inherit', cwd: PACKAGE_DIR });
+
+  // 2. Mirror templates
+  console.log('\n  Mirroring templates...');
+  mirrorTemplates(projectPath);
+
+  // 3. Get running container ID
+  const container = execSync('docker compose ps -q event-handler', {
+    encoding: 'utf8',
+    cwd: projectPath,
+  }).trim();
+
+  if (!container) {
+    console.error('\n  event-handler container is not running. Use full sync instead.\n');
+    process.exit(1);
+  }
+
+  // 4. Copy package source into container's node_modules/thepopebot/
+  const PKG_DEST = '/app/node_modules/thepopebot';
+  const PACKAGE_DIRS = ['lib', 'api', 'config'];
+
+  console.log('\n  Copying package source into container...');
+  for (const dir of PACKAGE_DIRS) {
+    execSync(`docker exec ${container} rm -rf ${PKG_DEST}/${dir}`, { stdio: 'inherit' });
+    execSync(`docker cp ${path.join(PACKAGE_DIR, dir)} ${container}:${PKG_DEST}/${dir}`, { stdio: 'inherit' });
+  }
+  // Also copy package.json for exports resolution
+  execSync(`docker cp ${path.join(PACKAGE_DIR, 'package.json')} ${container}:${PKG_DEST}/package.json`, { stdio: 'inherit' });
+
+  // 5. Copy web/app/ source into container for next build
+  const webDir = path.join(PACKAGE_DIR, 'web');
+  console.log('\n  Copying web source into container...');
+  execSync(`docker cp ${path.join(webDir, 'app')} ${container}:/app/app`, { stdio: 'inherit' });
+  execSync(`docker cp ${path.join(webDir, 'postcss.config.mjs')} ${container}:/app/postcss.config.mjs`, { stdio: 'inherit' });
+  execSync(`docker cp ${path.join(webDir, 'next.config.mjs')} ${container}:/app/next.config.mjs`, { stdio: 'inherit' });
+
+  // 6. Run next build inside the container
+  // Hide data/logs dirs so webpack's FileSystemInfo doesn't crawl them (causes OOM/RangeError
+  // when workspaces contain thousands of files). Restored immediately after build.
+  console.log('\n  Building Next.js inside container...');
+  execSync(`docker exec ${container} sh -c 'mv /app/data /app/.data-build-tmp 2>/dev/null; mv /app/logs /app/.logs-build-tmp 2>/dev/null; true'`, { stdio: 'inherit' });
+  try {
+    execSync(`docker exec ${container} ./node_modules/.bin/next build`, { stdio: 'inherit' });
+  } finally {
+    execSync(`docker exec ${container} sh -c 'mv /app/.data-build-tmp /app/data 2>/dev/null; mv /app/.logs-build-tmp /app/logs 2>/dev/null; true'`, { stdio: 'inherit' });
+  }
+
+  // 7. Clean up web source from container (not needed at runtime)
+  execSync(`docker exec ${container} rm -rf /app/app`, { stdio: 'inherit' });
+
+  // 8. Restart PM2
+  console.log('\n  Restarting server...');
+  execSync(`docker exec ${container} pm2 restart all`, { stdio: 'inherit' });
+
+  console.log('\n  Fast synced!\n');
+}
+
 export async function sync(projectPath) {
   if (!projectPath) {
     console.error('\n  Usage: thepopebot sync <path-to-project>\n');

package/config/instrumentation.js

@@ -70,5 +70,9 @@ export async function register() {
   const { startClusterRuntime } = await import('../lib/cluster/runtime.js');
   startClusterRuntime();
 
+  // Start internal maintenance cron (cleanup orphaned agent job keys, etc.)
+  const { startMaintenanceCron } = await import('../lib/maintenance.js');
+  startMaintenanceCron();
+
   console.log('thepopebot initialized');
 }

package/lib/ai/async-channel.js

@@ -0,0 +1,51 @@
+/**
+ * Async push/pull queue. Producer calls push()/done(), consumer uses for-await.
+ */
+export function createChannel() {
+  const queue = [];
+  const waiters = [];
+  let isDone = false;
+
+  return {
+    push(value) {
+      if (waiters.length > 0) waiters.shift()(value);
+      else queue.push(value);
+    },
+    done() {
+      isDone = true;
+      while (waiters.length > 0) waiters.shift()(Symbol.for('done'));
+    },
+    async *[Symbol.asyncIterator]() {
+      while (true) {
+        if (queue.length > 0) {
+          yield queue.shift();
+        } else if (isDone) {
+          return;
+        } else {
+          const value = await new Promise(resolve => waiters.push(resolve));
+          if (value === Symbol.for('done')) return;
+          yield value;
+        }
+      }
+    }
+  };
+}
+
+/**
+ * Merge two async iterables — yields from whichever has data first.
+ * Completes when BOTH are exhausted.
+ */
+export async function* mergeAsyncIterables(iter1, iter2) {
+  const channel = createChannel();
+  let active = 2;
+
+  const consume = async (iter) => {
+    for await (const item of iter) channel.push(item);
+    if (--active === 0) channel.done();
+  };
+
+  consume(iter1);
+  consume(iter2);
+
+  yield* channel;
+}