polygram 0.38.2 → 0.38.4

package/lib/db.js

@@ -26,6 +26,48 @@ const {
 const {
   sanitizeCodexFaultProvenance,
 } = require('./codex/fault-provenance');
+const {
+  sanitizeForDurableWrite,
+  sanitizeDurableStructured,
+  sanitizeDurableJsonText,
+} = require('./secret-detect');
+const {
+  enforceEventDetailSchema,
+} = require('./db/event-detail-schema');
+
+// Pre-write secret boundary. Every durable text sink below routes its text
+// through this so a recognized credential never reaches a row, the
+// external-content FTS index that mirrors it, a persisted error, or telemetry.
+// The caller's own value is untouched: the live provider turn always sees the
+// original message. Detection is deterministic shape/keyword matching, so
+// prose that never declares a credential is not covered here — the sweep and
+// the agent-reported redaction remain the defense-in-depth behind it.
+const durableText = (value) => (
+  typeof value === 'string' ? sanitizeForDurableWrite(value).text : value
+);
+
+// Truncating a masked text can cut a placeholder in half, leaving a fragment
+// like `‹redacted:kv-sec` that reads as ordinary text and no longer survives
+// a second masking pass unchanged. Cut before the placeholder that straddles
+// the limit instead.
+const truncateAfterMask = (text, limit) => {
+  if (typeof text !== 'string' || text.length <= limit) return text;
+  const cut = text.slice(0, limit);
+  const open = cut.lastIndexOf('‹');
+  if (open !== -1 && cut.indexOf('›', open) === -1) return cut.slice(0, open);
+  return cut;
+};
+
+// A JSON-valued column is sanitized as STRUCTURE, not as text: masking a
+// serialized document can splice across its delimiters, and an escaped quote
+// hides a declared value from the detector. Callers usually hand over an
+// object; a string is parsed back so a caller that pre-serialized still gets
+// the structural pass. Text that is not JSON falls back to string masking.
+const durableJson = (value) => {
+  if (value == null) return value;
+  if (typeof value !== 'string') return JSON.stringify(sanitizeDurableStructured(value));
+  return sanitizeDurableJsonText(value).text;
+};
 
 // 0.8.0 Phase 1: bumped from 9 → 10. Adds migration
 // 010-tool-use-id.sql (pending_approvals.tool_use_id column for the
@@ -45,7 +87,7 @@ const {
 //
 // 0.15: bumped 13 → 14. Adds migration 014-secret-redactions.sql
 // (secret_redactions audit table + messages.secret_scanned_at).
-const SCHEMA_VERSION = 19;
+const SCHEMA_VERSION = 20;
 
 // Sentinel `error` value for outbound rows whose API call may or may not
 // have reached Telegram. markStalePending writes it; hasOutboundReplyTo
@@ -1421,10 +1463,20 @@ function wrap(db) {
           itemId,
           detailJson,
         })) {
-          return { changes: 0, attemptId, kind };
+          const deferred = (
+            kind === 'telegram-delivery-failed'
+            && isCleanRetirementDelivery(attempt)
+          );
+          return {
+            changes: 0,
+            attemptId,
+            kind,
+            ...(deferred ? { deferred: true } : {}),
+          };
         }
       }
       assertCodexCheckpointOwner(generation, kind);
+      let deferred = false;
 
       if (kind === 'request-prepared') {
         requiredCodexString(method, 'request method');
@@ -1728,21 +1780,24 @@ function wrap(db) {
           );
         }
         const deliveryFailed = kind === 'telegram-delivery-failed';
-        const linkedState = deliveryFailed
-          ? 'failed'
-          : attempt.terminal_status === 'interrupted'
-            ? 'interrupted'
-            : attempt.terminal_status === 'failed'
-              ? 'failed'
-              : 'settled';
-        settleCodexAttemptConsumerOutcome({
-          attempt,
-          generationId,
-          linkedState,
-          attemptRecoveryState: 'settled',
-          settleQueuedReservation: true,
-          ts,
-        });
+        deferred = deliveryFailed && isCleanRetirementDelivery(attempt);
+        if (!deferred) {
+          const linkedState = deliveryFailed
+            ? 'failed'
+            : attempt.terminal_status === 'interrupted'
+              ? 'interrupted'
+              : attempt.terminal_status === 'failed'
+                ? 'failed'
+                : 'settled';
+          settleCodexAttemptConsumerOutcome({
+            attempt,
+            generationId,
+            linkedState,
+            attemptRecoveryState: 'settled',
+            settleQueuedReservation: true,
+            ts,
+          });
+        }
       } else if (
         ['active-start-cancelled', 'queued-send-cancelled'].includes(kind)
       ) {
@@ -1902,7 +1957,166 @@ function wrap(db) {
         detailJson,
         ts,
       );
-      return { changes: 1, attemptId, kind };
+      return {
+        changes: 1,
+        attemptId,
+        kind,
+        ...(deferred ? { deferred: true } : {}),
+      };
+    })();
+  }
+
+  function isCleanRetirementDelivery(attempt) {
+    if (
+      attempt?.method !== 'turn/start'
+      || attempt.terminal_status !== 'interrupted'
+      || !['terminal-pending', 'clean-pending'].includes(
+        attempt.recovery_state,
+      )
+    ) return false;
+    return db.prepare(`
+      SELECT 1
+        FROM codex_attempt_checkpoints
+       WHERE generation_id = ?
+         AND attempt_id = ?
+         AND kind = 'clean-retirement-requested'
+         AND thread_id = ?
+         AND turn_id = ?
+       LIMIT 1
+    `).get(
+      attempt.generation_id,
+      attempt.attempt_id,
+      attempt.thread_id,
+      attempt.turn_id,
+    ) != null;
+  }
+
+  function prepareCodexCleanRetirement(input) {
+    const allowed = new Set([
+      'generation_id',
+      'session_key',
+      'attempt_id',
+      'provider_session_id',
+      'provider_turn_id',
+      'source_message_id',
+      'stable_host_id',
+      'boot_session_id',
+      'ts',
+    ]);
+    assertExactKeys(
+      input,
+      allowed,
+      'CODEX_RETIREMENT_PREPARATION_REJECTED',
+      'clean retirement preparation',
+    );
+    const generationId = requiredCodexString(
+      input?.generation_id,
+      'generation ID',
+    );
+    const sessionKey = requiredCodexString(input?.session_key, 'session key');
+    const attemptId = requiredCodexString(input?.attempt_id, 'attempt ID');
+    const providerSessionId = requiredCodexString(
+      input?.provider_session_id,
+      'provider session ID',
+    );
+    const providerTurnId = requiredCodexString(
+      input?.provider_turn_id,
+      'provider turn ID',
+    );
+    const sourceMessageId = requiredCodexString(
+      input?.source_message_id == null
+        ? input?.source_message_id
+        : String(input.source_message_id),
+      'source message ID',
+    );
+    const stableHostId = requiredCodexString(
+      input?.stable_host_id,
+      'stable host identity',
+    );
+    const bootSessionId = requiredCodexString(
+      input?.boot_session_id,
+      'boot-session identity',
+    );
+    const ts = requiredTimestamp(
+      input?.ts ?? Date.now(),
+      'clean retirement preparation timestamp',
+    );
+
+    return db.transaction(() => {
+      const generation = assertCodexGenerationIdentity(
+        generationId,
+        stableHostId,
+        bootSessionId,
+      );
+      assertCodexCheckpointOwner(generation, 'clean-retirement-requested');
+      const attempt = requireCodexAttempt(attemptId, generationId);
+      const accepted = hasCodexTurnCheckpoint(
+        generationId,
+        attemptId,
+        'turn-accepted',
+        providerTurnId,
+      );
+      if (
+        generation.session_key !== sessionKey
+        || attempt.session_key !== sessionKey
+        || attempt.method !== 'turn/start'
+        || attempt.thread_id !== providerSessionId
+        || attempt.turn_id !== providerTurnId
+        || attempt.telegram_source_message_id !== sourceMessageId
+        || attempt.delivery_state !== 'response-observed'
+        || attempt.response_outcome !== 'result'
+        || attempt.recovery_state !== 'active'
+        || attempt.terminal_status != null
+        || !accepted
+      ) {
+        throw codexError(
+          'Codex clean retirement target is not the exact active accepted turn',
+          'CODEX_RETIREMENT_PREPARATION_REJECTED',
+        );
+      }
+      const existing = db.prepare(`
+        SELECT thread_id, turn_id
+          FROM codex_attempt_checkpoints
+         WHERE generation_id = ?
+           AND attempt_id = ?
+           AND kind = 'clean-retirement-requested'
+      `).all(generationId, attemptId);
+      if (existing.length > 0) {
+        if (
+          existing.length === 1
+          && existing[0].thread_id === providerSessionId
+          && existing[0].turn_id === providerTurnId
+        ) {
+          return {
+            changes: 0,
+            disposition: 'retirement-requested',
+            generationId,
+            attemptId,
+          };
+        }
+        throw codexError(
+          'Codex clean retirement target has conflicting ownership',
+          'CODEX_RETIREMENT_PREPARATION_REJECTED',
+        );
+      }
+      db.prepare(`
+        INSERT INTO codex_attempt_checkpoints (
+          generation_id, attempt_id, kind, thread_id, turn_id,
+          request_id, item_id, detail_json, ts
+        ) VALUES (?, ?, 'clean-retirement-requested', ?, ?, NULL, NULL, NULL, ?)
+      `).run(
+        generationId,
+        attemptId,
+        providerSessionId,
+        providerTurnId,
+        ts,
+      );
+      return {
+        changes: 1,
+        disposition: 'retirement-requested',
+        generationId,
+        attemptId,
+      };
     })();
   }
 
@@ -1930,7 +2144,7 @@ function wrap(db) {
     }
     return db.transaction(() => {
       const result = recordCodexCheckpoint(checkpoint);
-      if (!input.retireGeneration) {
+      if (!input.retireGeneration || result.deferred === true) {
         return { ...result, retired: false };
       }
       const generationId = requiredCodexString(
@@ -3788,7 +4002,7 @@ function wrap(db) {
         msg_id: row.msg_id,
         user: row.user || null,
         user_id: row.user_id || null,
-        text: row.text || '',
+        text: durableText(row.text || ''),
         reply_to_id: row.reply_to_id || null,
         direction: row.direction || 'in',
         source: row.source || 'polygram',
@@ -3798,7 +4012,7 @@ function wrap(db) {
         effort: row.effort || null,
         turn_id: row.turn_id || null,
         status: row.status || 'received',
-        error: row.error || null,
+        error: durableText(row.error) || null,
         cost_usd: row.cost_usd ?? null,
         ts: row.ts || Date.now(),
       });
@@ -3809,7 +4023,7 @@ function wrap(db) {
         chat_id: String(row.chat_id),
         thread_id: row.thread_id ? String(row.thread_id) : null,
         user: row.user || null,
-        text: row.text || '',
+        text: durableText(row.text || ''),
         source: row.source || 'polygram',
         bot_name: row.bot_name || null,
         turn_id: row.turn_id || null,
@@ -3825,7 +4039,10 @@ function wrap(db) {
     },
 
     markOutboundFailed(id, err) {
-      return markOutboundFailedStmt.run({ id, error: String(err).slice(0, 500) });
+      return markOutboundFailedStmt.run({
+        id,
+        error: truncateAfterMask(durableText(String(err)), 500),
+      });
     },
 
     updateOutboundText({ chat_id, msg_id, text, ts }) {
@@ -3833,7 +4050,7 @@ function wrap(db) {
       return updateOutboundTextStmt.run({
         chat_id: String(chat_id),
         msg_id: Number(msg_id),
-        text,
+        text: durableText(text),
         ts: ts || Date.now(),
       });
     },
@@ -3898,6 +4115,7 @@ function wrap(db) {
     },
 
     createCodexGeneration,
+    prepareCodexCleanRetirement,
     recordCodexCheckpoint,
     recordCodexDeliveryCheckpoint,
     getCodexAttempt(attemptId) {
@@ -3963,7 +4181,7 @@ function wrap(db) {
       return setMessageTextStmt.run({
         chat_id: String(chat_id),
         msg_id,
-        text: text ?? '',
+        text: durableText(text ?? ''),
       });
     },
 
@@ -3976,16 +4194,28 @@ function wrap(db) {
       return row?.new_chat_id || String(chatId);
     },
 
+    // Telemetry passes two gates before it is persisted: the content-free
+    // field schema decides WHICH fields may exist at all, then the durable
+    // secret boundary masks recognized credentials in what survived. The
+    // schema is the primary guarantee — masking alone would only remove
+    // content a detector recognizes.
     logEvent(kind, { chat_id = null, ...detail } = {}) {
+      const { detail: allowed, dropped, droppedCount } = enforceEventDetailSchema(detail);
+      // The count is always safe to record; the names only when they came
+      // from the schema's own vocabulary (an unknown key can be content).
+      if (droppedCount) allowed.dropped_field_count = droppedCount;
+      if (dropped.length) allowed.dropped_fields = dropped;
       return logEventStmt.run(
         Date.now(),
         chat_id ? String(chat_id) : null,
         kind,
-        Object.keys(detail).length ? JSON.stringify(detail) : null,
+        Object.keys(allowed).length
+          ? JSON.stringify(sanitizeDurableStructured(allowed))
+          : null,
       );
     },
 
-    // rc.61: find compact-command events from the recent window that
+    // Find compact-command events from the recent window that
     // never produced a matching compact-boundary on the same
     // session_key. These are "interrupted by deploy" cases — polygram
     // pushed /compact into SDK input but the daemon restarted before
@@ -3997,23 +4227,38 @@ function wrap(db) {
     // session_key AND ts > compact_command.ts. If none within
     // search-window, it's orphaned.
     //
+    // The command text itself is message content, so telemetry carries only
+    // the id of the inbound message that holds it. The text is joined back
+    // from that row — already sanitized by the durable-write boundary — and
+    // is null when the row is gone or the event predates the id, which the
+    // caller treats as "surface a retry notice instead of replaying".
+    //
+    // Events written before the hint moved out of telemetry still carry it in
+    // their detail. Those stay recoverable, from the row when one exists and
+    // otherwise from the stored copy, sanitized on the way out so a legacy
+    // row cannot hand a credential back.
+    //
     // @param {object} opts
     // @param {number} opts.olderThanMs - cutoff (only events newer than this are scanned)
-    // @returns {Array<{ts, chat_id, thread_id, session_key, user, user_id, text_len}>}
+    // @returns {Array<{ts, chat_id, thread_id, session_key, user, user_id, text_len, text}>}
     findOrphanedCompactCommands({ olderThanMs = 30 * 60 * 1000 } = {}) {
       const cutoff = Date.now() - olderThanMs;
       const cmds = db.prepare(`
-        SELECT id, ts, chat_id,
-               json_extract(detail_json, '$.thread_id')   AS thread_id,
-               json_extract(detail_json, '$.session_key') AS session_key,
-               json_extract(detail_json, '$.user')        AS user,
-               json_extract(detail_json, '$.user_id')     AS user_id,
-               json_extract(detail_json, '$.text_len')    AS text_len,
-               json_extract(detail_json, '$.text')        AS text
-          FROM events
-         WHERE kind = 'compact-command'
-           AND ts > ?
-         ORDER BY ts ASC, id ASC
+        SELECT event.id, event.ts, event.chat_id,
+               json_extract(event.detail_json, '$.thread_id')   AS thread_id,
+               json_extract(event.detail_json, '$.session_key') AS session_key,
+               json_extract(event.detail_json, '$.user_id')     AS user_id,
+               json_extract(event.detail_json, '$.text_len')    AS text_len,
+               source.text                                      AS row_text,
+               json_extract(event.detail_json, '$.text')        AS legacy_text
+          FROM events AS event
+          LEFT JOIN messages AS source
+            ON source.chat_id = event.chat_id
+           AND source.msg_id = json_extract(event.detail_json, '$.msg_id')
+           AND source.direction = 'in'
+         WHERE event.kind = 'compact-command'
+           AND event.ts > ?
+         ORDER BY event.ts ASC, event.id ASC
       `).all(cutoff);
       const orphans = [];
       for (const c of cmds) {
@@ -4030,7 +4275,7 @@ function wrap(db) {
            LIMIT 1
         `).get(c.id, c.session_key);
         if (boundary) continue;
-        // rc.66: also skip if a previous boot has already handled
+        // Also skip if a previous boot has already handled
         // this orphan (silent replay via compact-replay event, OR
         // surface-fallback via compact-failed-restart event). Both
         // of those record `original_ts` in their detail_json
@@ -4044,7 +4289,15 @@ function wrap(db) {
            LIMIT 1
         `).get(c.ts);
         if (handled) continue;
-        orphans.push(c);
+        const { row_text, legacy_text, ...rest } = c;
+        // Both sources are sanitized on the way out. The row is normally
+        // masked at write, but rows written before this boundary existed are
+        // not — and those are exactly the oldest, most likely to be replayed.
+        const recovered = row_text ?? legacy_text;
+        orphans.push({
+          ...rest,
+          text: recovered != null ? durableText(recovered) : null,
+        });
       }
       return orphans;
     },
@@ -4068,7 +4321,8 @@ function wrap(db) {
         num_assistant_messages: row.num_assistant_messages ?? null,
         num_tool_uses: row.num_tool_uses ?? null,
         result_subtype: row.result_subtype || null,
-        error: row.error || null,
+        // The turn's failure text can echo the provider's own output.
+        error: durableText(row.error) || null,
       });
     },
 
@@ -4111,7 +4365,12 @@ function wrap(db) {
         chat_id: String(row.chat_id),
         tool_name: String(row.tool_name),
         match_type: row.match_type,
-        input_pattern: row.input_pattern,
+        // The pattern is derived from a tool input, so it can carry a
+        // credential into durable storage. Masking it means a rule whose
+        // pattern contained one no longer matches that live input: the call
+        // falls through to a fresh approval prompt rather than silently
+        // auto-allowing or auto-denying it.
+        input_pattern: durableText(row.input_pattern),
         decision: row.decision,
         issued_ts: row.issued_ts || Date.now(),
         issued_by_user_id: row.issued_by_user_id != null
@@ -4558,7 +4817,6 @@ function wrap(db) {
     redactSecretInChat({ chat_id, thread_id = null, secret, now = Date.now(), limit = 200 }) {
       if (typeof secret !== 'string' || secret.length < 3) return { redacted: 0 };
       const PLACEHOLDER = '‹redacted:reported›';
-      const sha = require('crypto').createHash('sha256').update(secret).digest('hex');
       const rows = (thread_id != null
         ? db.prepare(`SELECT id, msg_id, text FROM messages WHERE chat_id=? AND thread_id=? AND direction='in' ORDER BY id DESC LIMIT ?`).all(String(chat_id), String(thread_id), limit)
         : db.prepare(`SELECT id, msg_id, text FROM messages WHERE chat_id=? AND direction='in' ORDER BY id DESC LIMIT ?`).all(String(chat_id), limit));
@@ -4568,8 +4826,8 @@ function wrap(db) {
           if (!r.text || !r.text.includes(secret)) continue;
           const newText = r.text.split(secret).join(PLACEHOLDER);
           db.prepare('UPDATE messages SET text = ? WHERE id = ?').run(newText, r.id);
-          db.prepare(`INSERT INTO secret_redactions (chat_id, msg_id, rule, tier, length, sha256, action, ts)
-                      VALUES (?,?,?,?,?,?,?,?)`).run(String(chat_id), r.msg_id, 'reported', 'reported', secret.length, sha, 'redacted', now);
+          db.prepare(`INSERT INTO secret_redactions (chat_id, msg_id, rule, tier, length, action, ts)
+                      VALUES (?,?,?,?,?,?,?)`).run(String(chat_id), r.msg_id, 'reported', 'reported', secret.length, 'redacted', now);
           redacted += 1;
         }
       });
@@ -4605,7 +4863,8 @@ function wrap(db) {
         file_id,
         file_unique_id || null,
         kind,
-        name || null,
+        // A user-supplied filename can carry a declared credential.
+        durableText(name) || null,
         mime_type || null,
         size_bytes ?? null,
         ts || Date.now(),
@@ -4629,13 +4888,13 @@ function wrap(db) {
            SET download_status = 'failed',
                download_error = ?
          WHERE id = ?
-      `).run(String(error || 'unknown').slice(0, 500), id);
+      `).run(truncateAfterMask(durableText(String(error || 'unknown')), 500), id);
     },
 
     setAttachmentTranscription(id, text) {
       return db.prepare(`
         UPDATE attachments SET transcription = ? WHERE id = ?
-      `).run(text || null, id);
+      `).run(durableJson(text) || null, id);
     },
 
     getAttachmentsByMessage(message_id) {

package/lib/handlers/abort.js

@@ -69,6 +69,10 @@ function createHandleAbort({
   isAbortRequest,
   markSessionAborted,
   clearAutosteeredReactions,
+  // Narrow async hook: retire the question state for a session whose process
+  // is being stopped. Kept as a callback so this handler never learns about
+  // the question store.
+  retireQuestionSession = null,
   getSessionKey,
   botName,
   // Cancel-cheap (spec Finding 5): delay before the second background-shell
@@ -146,6 +150,12 @@ function createHandleAbort({
         );
       }
 
+      // Only after the stop is proven, and only ever cleanup: Codex cannot
+      // raise an interactive question, so this closes a row a Claude turn left
+      // behind on a session that has since moved to Codex. An unverified stop
+      // changes nothing, exactly as before.
+      if (stopVerified) await retireQuestionSession?.(sessionKey);
+
       logEvent('abort-requested', {
         chat_id: chatId,
         user_id: msg.from?.id || null,
@@ -160,7 +170,8 @@ function createHandleAbort({
         feedback_clear_error_code: feedbackClearErrorCode,
         stop_verified: stopVerified,
         stop_error_code: stopErrorCode,
-        trigger: cleanText.slice(0, 40),
+        // The stop phrasing is the user's own message; its size is the signal.
+        text_len: cleanText.length,
       });
 
       // 👍 means the complete native stop proof settled, not merely that an
@@ -301,6 +312,9 @@ function createHandleAbort({
     }
 
     clearAutosteeredReactions(sessionKey).catch(() => {});
+    // The stopped turn owned any open question; retire its state with the same
+    // operation every other retirement path uses.
+    await retireQuestionSession?.(sessionKey);
     logEvent('abort-requested', {
       chat_id: chatId, user_id: msg.from?.id || null,
       had_active: hadActive,
@@ -320,9 +334,12 @@ function createHandleAbort({
         in_flight: busyProbe.inFlight,
         pending_turns: busyProbe.pendingTurns,
         captured: busyProbe.captured,
-        pane_tail: busyProbe.paneTail,
+        // The pane tail is live TUI transcript; only whether one was
+        // captured is recorded, never its content.
       } : null,
-      trigger: cleanText.slice(0, 40),
+      // The stop phrasing is the user's own message; its length is enough to
+      // tell a bare "stop" from a sentence that happened to contain one.
+      text_len: cleanText.length,
     });
 
     // Ack (locked design 2026-06-12, Ivan): a 👍 reaction on the user's stop

package/lib/handlers/approvals.js

@@ -243,7 +243,7 @@ function createApprovals({
       return;
     }
     logEvent('approval-resolved', {
-      id, status, by: userId, user, bot: botName,
+      id, status, by: userId, bot: botName,
     });
 
     // Edit the card to show the decision.
@@ -309,7 +309,7 @@ function createApprovals({
         rows = approvals.sweepTimedOut();
       } catch (err) {
         logger.error?.(`[approvals] sweeper DB error: ${err.message}`);
-        logEvent('approval-sweep-failed', { error: err.message?.slice(0, 300) });
+        logEvent('approval-sweep-failed', { error_class: err.name || 'Error' });
         return;
       }
       for (const row of rows) {

package/lib/handlers/config-callback.js

@@ -605,19 +605,19 @@ function createHandleConfigCallback({
         for (const { scope, value } of cleared) scope[configField] = value;
       };
       // The user tapped one card; topics they never touched can change value
-      // with it. Record which ones and what they held, so a config that moved
-      // on its own is still explicable afterwards. Logged only once the sweep
-      // is persisted — an event for a rolled-back sweep is worse than none.
+      // with it. Record WHICH ones, by id, so a config that moved on its own
+      // is still explicable afterwards — the values they held are
+      // configuration content and the ids answer the question without them.
+      // Logged only once the sweep is persisted — an event for a rolled-back
+      // sweep is worse than none.
       const logTopicRichTextSweep = (cleared) => {
         if (!cleared.length) return;
         dbWrite(() => {
           db.logEvent('config-topic-override-swept', {
             chat_id: chatId,
             field: configField,
-            topics: cleared.map(({ threadId, value }) => ({
-              thread_id: threadId,
-              old_value: value,
-            })),
+            topic_ids: cleared.map(({ threadId }) => String(threadId)),
+            topics_cleared: cleared.length,
           });
         }, 'log topic-override sweep');
       };