polygram 0.5.10 → 0.6.0

package/lib/db.js

@@ -8,7 +8,7 @@ const fs = require('fs');
 const path = require('path');
 const Database = require('better-sqlite3');
 
-const SCHEMA_VERSION = 6;
+const SCHEMA_VERSION = 7;
 
 function open(dbPath) {
   const db = new Database(dbPath);
@@ -361,6 +361,131 @@ function wrap(db) {
            AND ts > ?
       `).run(botName, cutoff);
     },
+
+    // ─── Attachments (migration 007, polygram 0.6.0) ──────────────────
+    //
+    // Replaces the messages.attachments_json blob. Each attachment is its
+    // own row with lifecycle (`pending` → `downloaded` | `failed`),
+    // searchable by chat / kind / time. recordInbound now inserts these
+    // alongside the message in a transaction; downloadAttachments updates
+    // status as it processes each file. See docs/attachments-table.md.
+
+    insertAttachment({
+      message_id, chat_id, msg_id, thread_id, bot_name,
+      file_id, file_unique_id, kind, name, mime_type, size_bytes,
+      ts,
+    }) {
+      return db.prepare(`
+        INSERT INTO attachments (
+          message_id, chat_id, msg_id, thread_id, bot_name,
+          file_id, file_unique_id, kind, name, mime_type, size_bytes,
+          download_status, ts
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?)
+      `).run(
+        message_id,
+        String(chat_id),
+        msg_id,
+        thread_id ? String(thread_id) : null,
+        bot_name || null,
+        file_id,
+        file_unique_id || null,
+        kind,
+        name || null,
+        mime_type || null,
+        size_bytes ?? null,
+        ts || Date.now(),
+      );
+    },
+
+    markAttachmentDownloaded(id, { local_path, size_bytes }) {
+      return db.prepare(`
+        UPDATE attachments
+           SET download_status = 'downloaded',
+               local_path = ?,
+               size_bytes = COALESCE(?, size_bytes),
+               download_error = NULL
+         WHERE id = ?
+      `).run(local_path, size_bytes ?? null, id);
+    },
+
+    markAttachmentFailed(id, error) {
+      return db.prepare(`
+        UPDATE attachments
+           SET download_status = 'failed',
+               download_error = ?
+         WHERE id = ?
+      `).run(String(error || 'unknown').slice(0, 500), id);
+    },
+
+    setAttachmentTranscription(id, text) {
+      return db.prepare(`
+        UPDATE attachments SET transcription = ? WHERE id = ?
+      `).run(text || null, id);
+    },
+
+    getAttachmentsByMessage(message_id) {
+      return db.prepare(`
+        SELECT id, message_id, chat_id, msg_id, thread_id, bot_name,
+               file_id, file_unique_id, kind, name, mime_type, size_bytes,
+               local_path, download_status, download_error, transcription, ts
+          FROM attachments
+         WHERE message_id = ?
+         ORDER BY id ASC
+      `).all(message_id);
+    },
+
+    // Rich filter for ops queries. All filters are optional; with no filters
+    // returns the most recent 100. Caller can paginate via since/until.
+    searchAttachments({
+      chat_id = null,
+      kind = null,
+      status = null,
+      since = null,
+      until = null,
+      limit = 100,
+    } = {}) {
+      const where = [];
+      const args = [];
+      if (chat_id !== null) { where.push('chat_id = ?'); args.push(String(chat_id)); }
+      if (kind !== null)    { where.push('kind = ?');    args.push(kind); }
+      if (status !== null)  { where.push('download_status = ?'); args.push(status); }
+      if (since !== null)   { where.push('ts >= ?');     args.push(Number(since)); }
+      if (until !== null)   { where.push('ts <= ?');     args.push(Number(until)); }
+      const sql = `
+        SELECT id, message_id, chat_id, msg_id, thread_id, bot_name,
+               file_id, file_unique_id, kind, name, mime_type, size_bytes,
+               local_path, download_status, download_error, transcription, ts
+          FROM attachments
+         ${where.length ? 'WHERE ' + where.join(' AND ') : ''}
+         ORDER BY ts DESC
+         LIMIT ?
+      `;
+      args.push(Number(limit));
+      return db.prepare(sql).all(...args);
+    },
+
+    // Look up the messages.id auto-pk for an inbound message. Used by
+    // recordInbound to FK attachments to the just-inserted message even
+    // when an ON-CONFLICT update happened (lastInsertRowid is 0 in that
+    // case, so we can't rely on the run-result alone).
+    getInboundMessageId({ chat_id, msg_id }) {
+      const row = db.prepare(`
+        SELECT id FROM messages WHERE chat_id = ? AND msg_id = ? AND direction = 'in'
+      `).get(String(chat_id), msg_id);
+      return row ? row.id : null;
+    },
+
+    listFailedAttachments({ since = null, limit = 100 } = {}) {
+      const cutoff = since ?? Date.now() - 24 * 60 * 60 * 1000;
+      return db.prepare(`
+        SELECT id, message_id, chat_id, msg_id, kind, name, mime_type,
+               download_error, ts
+          FROM attachments
+         WHERE download_status = 'failed' AND ts >= ?
+         ORDER BY ts DESC
+         LIMIT ?
+      `).all(cutoff, limit);
+    },
   };
 }
 

package/lib/prompt.js

@@ -121,9 +121,17 @@ function buildChannelAttrs({ chatId, msgId, user, userId, ts, threadId, topicNam
 
 function buildAttachmentTags(attachments) {
   if (!attachments?.length) return '';
-  return attachments.map((a) =>
-    `<attachment kind="${xmlEscape(a.kind)}" name="${xmlEscape(a.name)}" mime="${xmlEscape(a.mime_type)}" size="${a.size || 0}" path="${xmlEscape(a.path || '')}" />`
-  ).join('\n');
+  // Failed downloads (no `path`, has `error`) get a separate tag so claude
+  // can mention them to the user instead of pretending nothing was sent.
+  // The actual failure reason is included so claude can offer a useful
+  // recovery hint ("looks like the file is too large", "Telegram CDN had
+  // a 410 — could you resend?").
+  return attachments.map((a) => {
+    if (a.error || !a.path) {
+      return `<attachment-failed kind="${xmlEscape(a.kind)}" name="${xmlEscape(a.name)}" mime="${xmlEscape(a.mime_type)}" reason="${xmlEscape(a.error || 'no local path')}" />`;
+    }
+    return `<attachment kind="${xmlEscape(a.kind)}" name="${xmlEscape(a.name)}" mime="${xmlEscape(a.mime_type)}" size="${a.size || 0}" path="${xmlEscape(a.path)}" />`;
+  }).join('\n');
 }
 
 function buildVoiceTags(attachments) {

package/migrations/007-attachments-table.sql

@@ -0,0 +1,89 @@
+-- Replace the messages.attachments_json blob with a real table so we can
+-- query, search, and track lifecycle per attachment. Design doc:
+-- docs/attachments-table.md.
+--
+-- This migration creates the table + indexes and backfills from existing
+-- attachments_json. The column itself is NOT dropped here — kept as a
+-- safety net for one minor release. A follow-up migration drops it once
+-- we're confident reads/writes have fully moved over.
+--
+-- For backfilled rows we set download_status='downloaded' (we know they
+-- went through to disk historically; recreating failure state isn't
+-- useful for old data). Local path is left NULL — the deterministic
+-- on-disk location can be re-derived from msg_id + file_unique_id at
+-- read time if a caller needs it; the existing inbox/<chat_id>/...
+-- filename convention is unchanged.
+
+CREATE TABLE IF NOT EXISTS attachments (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  message_id      INTEGER NOT NULL,
+  chat_id         TEXT    NOT NULL,
+  msg_id          INTEGER NOT NULL,
+  thread_id       TEXT,
+  bot_name        TEXT,
+  file_id         TEXT    NOT NULL,
+  file_unique_id  TEXT,
+  kind            TEXT    NOT NULL,
+  name            TEXT,
+  mime_type       TEXT,
+  size_bytes      INTEGER,
+  local_path      TEXT,
+  download_status TEXT    NOT NULL CHECK(download_status IN ('pending','downloaded','failed')),
+  download_error  TEXT,
+  transcription   TEXT,
+  ts              INTEGER NOT NULL,
+  FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_attachments_chat_ts
+  ON attachments(chat_id, ts);
+
+CREATE INDEX IF NOT EXISTS idx_attachments_kind_ts
+  ON attachments(kind, ts);
+
+-- Narrow index: only the small set we actually want to query for retries
+-- and dashboards. Skips the bulk 'downloaded' rows.
+CREATE INDEX IF NOT EXISTS idx_attachments_status
+  ON attachments(download_status, ts)
+  WHERE download_status != 'downloaded';
+
+CREATE INDEX IF NOT EXISTS idx_attachments_message
+  ON attachments(message_id);
+
+-- Narrow: file_unique_id is NULL for some historical rows (Telegram doesn't
+-- always populate it for old messages); only index the populated ones.
+CREATE INDEX IF NOT EXISTS idx_attachments_unique_id
+  ON attachments(file_unique_id)
+  WHERE file_unique_id IS NOT NULL;
+
+-- Backfill from messages.attachments_json. Only rows that aren't already
+-- represented (idempotent — re-running this migration on a partially-
+-- migrated DB doesn't double-insert).
+--
+-- attachments_json shape: array of objects, each with kind, name,
+-- mime_type, size, file_id, file_unique_id, and optionally transcription.
+-- Pre-0.5.x rows may not have file_unique_id. We pull what's there and
+-- leave the rest NULL.
+INSERT INTO attachments (
+  message_id, chat_id, msg_id, thread_id, bot_name,
+  file_id, file_unique_id, kind, name, mime_type, size_bytes,
+  local_path, download_status, transcription, ts
+)
+SELECT
+  m.id, m.chat_id, m.msg_id, m.thread_id, m.bot_name,
+  COALESCE(json_extract(att.value, '$.file_id'), ''),
+  json_extract(att.value, '$.file_unique_id'),
+  COALESCE(json_extract(att.value, '$.kind'), 'document'),
+  json_extract(att.value, '$.name'),
+  json_extract(att.value, '$.mime_type'),
+  json_extract(att.value, '$.size'),
+  json_extract(att.value, '$.path'),
+  'downloaded',
+  json_extract(att.value, '$.transcription.text'),
+  m.ts
+FROM messages m, json_each(m.attachments_json) att
+WHERE m.attachments_json IS NOT NULL
+  AND m.direction = 'in'
+  AND NOT EXISTS (
+    SELECT 1 FROM attachments a WHERE a.message_id = m.id
+  );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "polygram",
-  "version": "0.5.10",
+  "version": "0.6.0",
   "description": "Telegram daemon for Claude Code that preserves the OpenClaw per-chat session model. Migration path for OpenClaw users moving to Claude Code.",
   "main": "lib/ipc-client.js",
   "bin": {

package/polygram.js

@@ -206,23 +206,52 @@ function recordInbound(msg) {
   const user = msg.from?.first_name || msg.from?.username || null;
   const attachments = extractAttachments(msg);
   const chatConfig = config.chats[chatId];
+  const ts = (msg.date || Math.floor(Date.now() / 1000)) * 1000;
 
-  dbWrite(() => db.insertMessage({
-    chat_id: chatId,
-    thread_id: threadId,
-    msg_id: msg.message_id,
-    user,
-    user_id: msg.from?.id || null,
-    text: msg.text || msg.caption || '',
-    reply_to_id: msg.reply_to_message?.message_id || null,
-    direction: 'in',
-    source: 'polygram',
-    bot_name: BOT_NAME,
-    attachments_json: attachments.length ? JSON.stringify(attachments) : null,
-    model: chatConfig?.model || null,
-    effort: chatConfig?.effort || null,
-    ts: (msg.date || Math.floor(Date.now() / 1000)) * 1000,
-  }), `insert inbound ${chatId}/${msg.message_id}`);
+  dbWrite(() => {
+    db.insertMessage({
+      chat_id: chatId,
+      thread_id: threadId,
+      msg_id: msg.message_id,
+      user,
+      user_id: msg.from?.id || null,
+      text: msg.text || msg.caption || '',
+      reply_to_id: msg.reply_to_message?.message_id || null,
+      direction: 'in',
+      source: 'polygram',
+      bot_name: BOT_NAME,
+      // attachments_json kept temporarily as a fallback during the 0.6.0
+      // migration window; per-attachment rows below are the source of
+      // truth. Will be dropped in a follow-up minor.
+      attachments_json: attachments.length ? JSON.stringify(attachments) : null,
+      model: chatConfig?.model || null,
+      effort: chatConfig?.effort || null,
+      ts,
+    });
+
+    if (!attachments.length) return;
+    // Look up the just-inserted (or ON-CONFLICT-updated) message row id
+    // so attachments can FK to it. lastInsertRowid is unreliable across
+    // the upsert path; an explicit lookup is cheap and always correct.
+    const messageId = db.getInboundMessageId({ chat_id: chatId, msg_id: msg.message_id });
+    if (!messageId) return;
+    for (const att of attachments) {
+      db.insertAttachment({
+        message_id: messageId,
+        chat_id: chatId,
+        msg_id: msg.message_id,
+        thread_id: threadId,
+        bot_name: BOT_NAME,
+        file_id: att.file_id,
+        file_unique_id: att.file_unique_id,
+        kind: att.kind,
+        name: att.name,
+        mime_type: att.mime_type,
+        size_bytes: att.size,
+        ts,
+      });
+    }
+  }, `insert inbound ${chatId}/${msg.message_id}`);
 }
 
 
@@ -344,30 +373,57 @@ async function transcribeVoiceAttachments(downloaded, { chatId, msgId, label, bo
     }
   }));
 
-  // Persist transcription into the inbound row so FTS search finds it.
-  // Combine all successful transcriptions into `text` and mirror the
-  // transcription data back into attachments_json.
+  // Persist transcription:
+  //   - Per-attachment: setAttachmentTranscription stores the full
+  //     transcription object (text + language + duration + provider) as
+  //     JSON in the attachments.transcription column. buildVoiceTags
+  //     parses it back when building the prompt.
+  //   - Message-level: setMessageText updates messages.text with the
+  //     combined transcript so FTS finds "what Maria said" via the
+  //     normal chat search path. attachments_json is left as-is (will
+  //     be dropped in a future minor; per-attachment row is the source
+  //     of truth).
   const successful = targets.filter((a) => a.transcription?.text);
   if (!successful.length) return;
+  for (const a of successful) {
+    if (a.id != null) {
+      dbWrite(() => db.setAttachmentTranscription(a.id, JSON.stringify(a.transcription)),
+        `setAttachmentTranscription ${a.id}`);
+    }
+  }
   const combinedText = successful.map((a) => a.transcription.text).join(' ').trim();
-  const attJson = JSON.stringify(downloaded.map((a) => ({
-    kind: a.kind, name: a.name, mime_type: a.mime_type, size: a.size,
-    path: a.path, file_unique_id: a.file_unique_id,
-    transcription: a.transcription || null,
-  })));
   dbWrite(() => db.setMessageText({
     chat_id: chatId, msg_id: msgId,
-    text: combinedText, attachments_json: attJson,
+    text: combinedText, attachments_json: null,
   }), 'persist voice transcription');
 }
 
-async function downloadAttachments(bot, token, chatId, msg, attachments) {
-  if (!attachments.length) return [];
+// 0.6.0: takes attachment ROW objects from the DB (not raw extracted
+// metadata). Each row has an `id` so we can mark status as we go.
+// On replay: a row with status='downloaded' and a local_path that's
+// still on disk is reused without re-fetching. Anything else (failed,
+// missing file, never downloaded) hits Telegram's CDN.
+async function downloadAttachments(bot, token, chatId, msg, rows) {
+  if (!rows.length) return [];
   const chatDir = path.join(INBOX_DIR, String(chatId));
   fs.mkdirSync(chatDir, { recursive: true });
 
   const results = [];
-  for (const att of attachments) {
+  for (const att of rows) {
+    // Reuse path: row already says downloaded AND the file is on disk.
+    if (att.download_status === 'downloaded' && att.local_path) {
+      try {
+        if (fs.statSync(att.local_path).size > 0) {
+          results.push({
+            ...att,
+            path: att.local_path,
+            size: att.size_bytes || 0,
+            error: null,
+          });
+          continue;
+        }
+      } catch { /* fall through to refetch */ }
+    }
     try {
       const fileInfo = await bot.api.getFile(att.file_id);
       if (!fileInfo?.file_path) throw new Error('no file_path from getFile');
@@ -412,10 +468,22 @@ async function downloadAttachments(bot, token, chatId, msg, attachments) {
           console.log(`[attach] ${chatId} ← ${att.kind} ${safeName} (race: already on disk)`);
         }
       }
-      results.push({ ...att, path: localPath, size: att.size || buf.length });
+      results.push({ ...att, path: localPath, size: att.size_bytes || buf.length, error: null });
       console.log(`[attach] ${chatId} ← ${att.kind} ${safeName} (${buf.length} bytes) → ${localPath}`);
+      dbWrite(() => db.markAttachmentDownloaded(att.id, {
+        local_path: localPath, size_bytes: att.size_bytes || buf.length,
+      }), `markAttachmentDownloaded ${att.id}`);
     } catch (err) {
-      console.error(`[attach] download failed for ${att.name}: ${err.message}`);
+      // Don't drop the attachment silently — push it through with the
+      // failure noted. buildAttachmentTags renders this as
+      // <attachment-failed reason="..." /> so claude tells the user
+      // "I couldn't see your <kind>" instead of pretending it received
+      // text only.
+      const reason = (err.message || 'unknown').slice(0, 200);
+      console.error(`[attach] download failed for ${att.name}: ${reason}`);
+      results.push({ ...att, path: null, error: reason });
+      dbWrite(() => db.markAttachmentFailed(att.id, reason),
+        `markAttachmentFailed ${att.id}`);
     }
   }
   return results;
@@ -1391,7 +1459,39 @@ async function handleMessage(sessionKey, chatId, msg, bot) {
     dbWrite(() => db.logEvent('attachment-skipped', { chat_id: chatId, msg_id: msg.message_id, name: att.name, reason }), 'log attachment-skipped');
   }
   const token = config.bot?.token || '';
-  const downloaded = accepted.length ? await downloadAttachments(bot, token, chatId, msg, accepted) : [];
+
+  // 0.6.0: pull persisted attachment rows (recordInbound inserted them
+  // upstream). Filter to the ones that survived filterAttachments.
+  // Replays / reconstructed messages may not have inserted rows yet —
+  // for that path we fall back to the in-memory `accepted` list. Both
+  // shapes have the same fields downloadAttachments consumes (kind,
+  // file_id, file_unique_id, name, mime_type) plus optionally `id` /
+  // `download_status` / `local_path` for the row variant.
+  const messageId = db.getInboundMessageId({ chat_id: chatId, msg_id: msg.message_id });
+  const allRows = messageId ? db.getAttachmentsByMessage(messageId) : [];
+  const acceptedKeys = new Set(accepted.map((a) => a.file_unique_id || a.file_id));
+  let downloadInputs;
+  if (allRows.length) {
+    downloadInputs = allRows.filter((r) => acceptedKeys.has(r.file_unique_id || r.file_id));
+  } else {
+    // Fallback for replayed turns where rows weren't persisted: synthesize
+    // row-like objects so downloadAttachments treats them as never-tried.
+    downloadInputs = accepted.map((a) => ({
+      ...a, id: null, size_bytes: a.size,
+      download_status: 'pending', local_path: null,
+    }));
+  }
+  const downloaded = downloadInputs.length
+    ? await downloadAttachments(bot, token, chatId, msg, downloadInputs)
+    : [];
+  // Decode JSON-encoded transcription on enriched rows so buildVoiceTags
+  // can read .text/.language/.duration_sec/.provider directly.
+  for (const a of downloaded) {
+    if (typeof a.transcription === 'string' && a.transcription) {
+      try { a.transcription = JSON.parse(a.transcription); }
+      catch { /* leave as string */ }
+    }
+  }
   if (rejected.length) {
     const summary = rejected.map(({ att, reason }) => `${att.name}: ${reason}`).join('; ');
     try {
@@ -2331,13 +2431,21 @@ async function main() {
           ...(row.thread_id && { message_thread_id: Number(row.thread_id) }),
           ...(row.reply_to_id && { reply_to_message: { message_id: row.reply_to_id } }),
         };
-        // Attach already-extracted attachments via the media-group shortcut
+        // Attach already-recorded attachments via the media-group shortcut
         // field so extractAttachments picks them up without re-parsing
         // grammy fields that don't exist on this reconstructed object.
-        if (row.attachments_json) {
-          try {
-            reconstructed._mergedAttachments = JSON.parse(row.attachments_json);
-          } catch {}
+        // 0.6.0: read from the per-attachment table; fall back to the
+        // legacy attachments_json blob for rows inserted before migration
+        // 007 ran (covers the small window during the upgrade).
+        const attRows = db.getAttachmentsByMessage(row.id);
+        if (attRows.length) {
+          reconstructed._mergedAttachments = attRows.map((a) => ({
+            kind: a.kind, name: a.name, mime_type: a.mime_type,
+            size: a.size_bytes, file_id: a.file_id, file_unique_id: a.file_unique_id,
+          }));
+        } else if (row.attachments_json) {
+          try { reconstructed._mergedAttachments = JSON.parse(row.attachments_json); }
+          catch {}
         }
         const chatConfig = config.chats[row.chat_id];
         if (!chatConfig) { skipped += 1; continue; }