talon-agent 3.34.1 → 3.35.0

package/src/frontend/whatsapp/message-store.ts

@@ -9,14 +9,22 @@
  * not just the id. So each message Talon sees — inbound or sent — is
  * assigned a monotonic numeric id here and remembered alongside its key.
  *
- * `forward_message` needs the full message, not only its key, so the
- * proto is retained too. The map is bounded and evicts oldest-first: a
- * long-running daemon must not accumulate every message it ever saw, and
- * a model addressing a message thousands of turns back is not a case
- * worth holding memory for.
+ * `forward_message` and media re-downloads need the full message, not
+ * only its key, so the proto is retained too. The in-memory map is a
+ * bounded cache that evicts oldest-first; the record is the
+ * `whatsapp_messages` table (storage/whatsapp-messages.ts), which a miss
+ * falls through to — so an id from before the last restart, or from
+ * thousands of messages back, still resolves.
  */
 
-import type { WAMessage, WAMessageKey } from "baileys";
+import { proto, type WAMessage, type WAMessageKey } from "baileys";
+import {
+  getWhatsAppMessage,
+  getWhatsAppMessageByWaId,
+  saveWhatsAppMessage,
+  clearWhatsAppMessages,
+  type WhatsAppMessageRecord,
+} from "../../storage/whatsapp-messages.js";
 
 export type StoredMessage = {
   /** Talon-facing numeric id. */
@@ -25,7 +33,7 @@ export type StoredMessage = {
   key: WAMessageKey;
   /** Talon chat id (wa_dm_… / wa_group_…). */
   chatId: string;
-  /** Retained for forward_message, which re-sends the whole message. */
+  /** Retained for forward_message and media re-downloads. */
   message?: WAMessage;
   /** Plain text at the time it was stored (copy_message, previews). */
   text: string;
@@ -43,16 +51,15 @@ const MAX_TRACKED = 2_000;
 let nextId = ID_BASE;
 
 /**
- * Raise the id counter past what persistent history already holds.
+ * Raise the id counter past what persistent storage already holds.
  *
  * The counter is in-memory and restarts at ID_BASE every boot, but the
- * ids it hands out are also the `msg_id`s written to the history table,
- * where `INSERT OR IGNORE` + UNIQUE(chat_id, msg_id) dedupes. Without
+ * ids it hands out are also the `msg_id`s written to the history and
+ * whatsapp_messages tables, where `INSERT OR IGNORE` dedupes. Without
  * this seed, the first messages after a daemon restart re-issue ids the
- * previous run already used — the IGNORE then silently drops them from
- * history, and a reaction/reply addressed at an old id from history hits
- * whatever new message reused the number. Called at frontend start with
- * max(msg_id) over wa_* chats + 1.
+ * previous run already used — the IGNORE then silently drops them, and
+ * a reaction/reply addressed at an old id hits whatever new message
+ * reused the number. Called at frontend start with max(msg_id) + 1.
  */
 export function seedMessageStore(floor: number): void {
   if (Number.isFinite(floor)) nextId = Math.max(nextId, Math.floor(floor));
@@ -71,10 +78,74 @@ function evictOldest(): void {
   }
 }
 
+function cache(stored: StoredMessage): void {
+  byMsgId.set(stored.msgId, stored);
+  if (stored.key.id) byWaId.set(stored.key.id, stored.msgId);
+  evictOldest();
+}
+
+/**
+ * Proto ↔ JSON. Byte fields (media keys, hashes) travel as base64:
+ * `toJSON` on a WebMessageInfo instance emits them that way, and
+ * `fromObject` decodes them back, so a rehydrated message downloads.
+ * Normalising through `fromObject` first also covers a plain object
+ * that was never a proto instance (a Uint8Array would otherwise
+ * stringify as `{"0":…}`).
+ */
+function serializeMessage(message: WAMessage): string {
+  return JSON.stringify(proto.WebMessageInfo.fromObject(message).toJSON());
+}
+
+/** Rebuild a stored message from its persisted row. */
+function fromRecord(record: WhatsAppMessageRecord): StoredMessage {
+  const key: WAMessageKey = {
+    id: record.waId,
+    remoteJid: record.remoteJid,
+    fromMe: record.fromMe,
+    ...(record.participant ? { participant: record.participant } : {}),
+  };
+  let message: WAMessage | undefined;
+  if (record.messageJson) {
+    try {
+      message = proto.WebMessageInfo.fromObject(
+        JSON.parse(record.messageJson),
+      ) as WAMessage;
+    } catch {
+      message = undefined; // an undecodable proto loses only forwarding
+    }
+  }
+  return {
+    msgId: record.msgId,
+    key,
+    chatId: record.chatId,
+    text: record.text,
+    senderName: record.senderName,
+    timestamp: record.timestamp,
+    ...(message ? { message } : {}),
+  };
+}
+
+function persist(stored: StoredMessage): void {
+  if (!stored.key.id || !stored.key.remoteJid) return;
+  saveWhatsAppMessage({
+    chatId: stored.chatId,
+    msgId: stored.msgId,
+    waId: stored.key.id,
+    remoteJid: stored.key.remoteJid,
+    fromMe: stored.key.fromMe === true,
+    participant: stored.key.participant ?? undefined,
+    senderName: stored.senderName,
+    text: stored.text,
+    timestamp: stored.timestamp,
+    messageJson: stored.message ? serializeMessage(stored.message) : undefined,
+  });
+}
+
 /**
  * Record a message and return its Talon numeric id. Re-recording a
  * WhatsApp id already seen returns the original number, so the same
- * message never gets two identities (Baileys can re-deliver on reconnect).
+ * message never gets two identities (Baileys can re-deliver on reconnect,
+ * and a restart in between must not change the answer).
  */
 export function rememberMessage(entry: {
   key: WAMessageKey;
@@ -86,7 +157,7 @@ export function rememberMessage(entry: {
 }): number {
   const waId = entry.key.id;
   if (waId) {
-    const existing = byWaId.get(waId);
+    const existing = byWaId.get(waId) ?? getWhatsAppMessageByWaId(waId)?.msgId;
     if (existing !== undefined) return existing;
   }
   const msgId = nextId++;
@@ -99,21 +170,26 @@ export function rememberMessage(entry: {
     timestamp: entry.timestamp ?? Date.now(),
     ...(entry.message ? { message: entry.message } : {}),
   };
-  byMsgId.set(msgId, stored);
-  if (waId) byWaId.set(waId, msgId);
-  evictOldest();
+  cache(stored);
+  persist(stored);
   return msgId;
 }
 
 /** Look up a stored message by its Talon numeric id. */
 export function lookupMessage(msgId: number): StoredMessage | undefined {
-  return byMsgId.get(msgId);
+  const cached = byMsgId.get(msgId);
+  if (cached) return cached;
+  const record = getWhatsAppMessage(msgId);
+  if (!record) return undefined;
+  const stored = fromRecord(record);
+  cache(stored);
+  return stored;
 }
 
 /** Look up by WhatsApp's own id (reply resolution on inbound). */
 export function lookupByWaId(waId: string): StoredMessage | undefined {
-  const msgId = byWaId.get(waId);
-  return msgId === undefined ? undefined : byMsgId.get(msgId);
+  const msgId = byWaId.get(waId) ?? getWhatsAppMessageByWaId(waId)?.msgId;
+  return msgId === undefined ? undefined : lookupMessage(msgId);
 }
 
 /**
@@ -134,12 +210,12 @@ export function resolveKey(
   if (!Number.isFinite(msgId)) {
     return { error: `Invalid message_id: ${String(value)}` };
   }
-  const stored = byMsgId.get(msgId);
+  const stored = lookupMessage(msgId);
   if (!stored) {
     return {
       error:
         `Unknown message_id ${msgId} — WhatsApp message ids are assigned ` +
-        `when Talon sees the message; only recent ones can be acted on.`,
+        `when Talon sees the message; use read_chat_history to find one.`,
     };
   }
   if (stored.chatId !== chatId) {
@@ -152,5 +228,6 @@ export function resolveKey(
 export function resetMessageStore(): void {
   byMsgId.clear();
   byWaId.clear();
+  clearWhatsAppMessages();
   nextId = ID_BASE;
 }

package/src/storage/history.ts

@@ -199,17 +199,30 @@ function chatIsEmpty(chatId: string): boolean {
   return repo.latestMsgId(chatId) === undefined;
 }
 
+/** Optional time window for `searchHistory`, in epoch ms. */
+export type HistoryDateRange = { after?: number; before?: number };
+
 export function searchHistory(
   chatId: string,
   query: string,
   limit = 20,
+  range: HistoryDateRange = {},
 ): string {
   if (chatIsEmpty(chatId)) return "No messages in history.";
   const match = ftsQuery(query);
   if (!match) return `No messages matching "${query}".`;
   let messages: HistoryMessage[];
   try {
-    messages = repo.searchFts(chatId, match, limit);
+    messages =
+      range.after === undefined && range.before === undefined
+        ? repo.searchFts(chatId, match, limit)
+        : repo.searchFtsBetween(
+            chatId,
+            match,
+            range.after ?? 0,
+            range.before ?? Number.MAX_SAFE_INTEGER,
+            limit,
+          );
   } catch (err) {
     logError("history", `FTS search failed for ${JSON.stringify(query)}`, err);
     return `No messages matching "${query}".`;
@@ -229,6 +242,14 @@ export function getMessagesByUser(
   return messages.map(formatMessage).join("\n");
 }
 
+/** The stored row for one message, or undefined. */
+export function getHistoryMessage(
+  chatId: string,
+  msgId: number,
+): HistoryMessage | undefined {
+  return repo.byMsgId(chatId, msgId);
+}
+
 export function getMessageById(chatId: string, msgId: number): string {
   if (chatIsEmpty(chatId)) return "No messages in history.";
   const msg = repo.byMsgId(chatId, msgId);

package/src/storage/repositories/history-repo.ts

@@ -147,6 +147,20 @@ export function searchFts(
   return rows.reverse().map(rowToMessage);
 }
 
+/** `searchFts` restricted to messages with `after <= timestamp < before`. */
+export function searchFtsBetween(
+  chatId: string,
+  match: string,
+  after: number,
+  before: number,
+  limit: number,
+): HistoryMessage[] {
+  const rows = getDatabase()
+    .prepare(historySql.searchFtsBetween)
+    .all(chatId, match, after, before, limit) as Row[];
+  return rows.reverse().map(rowToMessage);
+}
+
 export function bySenderName(
   chatId: string,
   nameFragment: string,

package/src/storage/repositories/whatsapp-messages-repo.ts

@@ -0,0 +1,101 @@
+/**
+ * WhatsApp message-key repository — executes the statements in
+ * sql/whatsapp-messages.sql against the `whatsapp_messages` table; no
+ * SQL text lives here. The public store (storage/whatsapp-messages.ts)
+ * holds the domain API; this module owns statement execution and the
+ * row↔domain mapping.
+ */
+
+import { getDatabase } from "../db.js";
+import { whatsappMessagesSql } from "../sql/statements.generated.js";
+
+/** One persisted WhatsApp message as the domain sees it. */
+export type WhatsAppMessageRecord = {
+  chatId: string;
+  msgId: number;
+  waId: string;
+  remoteJid: string;
+  fromMe: boolean;
+  participant?: string;
+  senderName: string;
+  text: string;
+  timestamp: number;
+  /** The full WAMessage proto as JSON, when it was retained. */
+  messageJson?: string;
+};
+
+type Row = {
+  chat_id: string;
+  msg_id: number;
+  wa_id: string;
+  remote_jid: string;
+  from_me: number;
+  participant: string | null;
+  sender_name: string;
+  text: string;
+  timestamp: number;
+  message_json: string | null;
+};
+
+function rowToRecord(row: Row): WhatsAppMessageRecord {
+  return {
+    chatId: row.chat_id,
+    msgId: row.msg_id,
+    waId: row.wa_id,
+    remoteJid: row.remote_jid,
+    fromMe: row.from_me === 1,
+    participant: row.participant ?? undefined,
+    senderName: row.sender_name,
+    text: row.text,
+    timestamp: row.timestamp,
+    messageJson: row.message_json ?? undefined,
+  };
+}
+
+export function insert(record: WhatsAppMessageRecord): void {
+  getDatabase()
+    .prepare(whatsappMessagesSql.insert)
+    .run(
+      record.chatId,
+      record.msgId,
+      record.waId,
+      record.remoteJid,
+      record.fromMe ? 1 : 0,
+      record.participant ?? null,
+      record.senderName,
+      record.text,
+      record.timestamp,
+      record.messageJson ?? null,
+    );
+}
+
+export function byMsgId(msgId: number): WhatsAppMessageRecord | undefined {
+  const row = getDatabase().prepare(whatsappMessagesSql.byMsgId).get(msgId) as
+    Row | undefined;
+  return row ? rowToRecord(row) : undefined;
+}
+
+export function byWaId(waId: string): WhatsAppMessageRecord | undefined {
+  const row = getDatabase().prepare(whatsappMessagesSql.byWaId).get(waId) as
+    Row | undefined;
+  return row ? rowToRecord(row) : undefined;
+}
+
+export function maxMsgId(): number | undefined {
+  const row = getDatabase().prepare(whatsappMessagesSql.maxMsgId).get() as {
+    max_id: number | null;
+  };
+  return row.max_id ?? undefined;
+}
+
+/** Delete rows older than `cutoff` (ms); returns the number removed. */
+export function deleteOlderThan(cutoff: number): number {
+  const result = getDatabase()
+    .prepare(whatsappMessagesSql.deleteOlderThan)
+    .run(cutoff) as { changes: number | bigint };
+  return Number(result.changes);
+}
+
+export function deleteAll(): void {
+  getDatabase().prepare(whatsappMessagesSql.deleteAll).run();
+}

package/src/storage/sql/history.sql

@@ -45,6 +45,18 @@ WHERE chat_id = ?
   AND id IN (SELECT rowid FROM history_fts WHERE history_fts MATCH ?)
 ORDER BY id DESC LIMIT ?
 
+-- name: searchFtsBetween
+-- searchFts restricted to a timestamp window [after, before) — the
+-- search_history `after` / `before` date parameters. Either bound may be
+-- the open end of the range (0 / a far-future value).
+SELECT msg_id, sender_id, sender_name, sender_handle, text, reply_to_msg_id,
+       timestamp, media_type, sticker_file_id, file_path
+FROM history_messages
+WHERE chat_id = ?
+  AND id IN (SELECT rowid FROM history_fts WHERE history_fts MATCH ?)
+  AND timestamp >= ? AND timestamp < ?
+ORDER BY id DESC LIMIT ?
+
 -- name: bySenderName
 -- The fragment param is LIKE-escaped by the repository (backslash escape).
 SELECT msg_id, sender_id, sender_name, sender_handle, text, reply_to_msg_id,

package/src/storage/sql/schema.sql

@@ -251,3 +251,24 @@ CREATE TABLE IF NOT EXISTS journal (
   payload TEXT    NOT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_journal_type ON journal(type, seq);
+
+-- WhatsApp message keys: Talon's numeric message id ↔ WhatsApp's
+-- (id, remoteJid, fromMe, participant) key, plus the full proto as JSON
+-- so react/reply/forward/download keep working on messages from before
+-- a restart and media past the CDN TTL can be re-requested.
+CREATE TABLE IF NOT EXISTS whatsapp_messages (
+  chat_id TEXT NOT NULL,
+  msg_id INTEGER NOT NULL,
+  wa_id TEXT NOT NULL,
+  remote_jid TEXT NOT NULL,
+  from_me INTEGER NOT NULL DEFAULT 0,
+  participant TEXT,
+  sender_name TEXT NOT NULL DEFAULT '',
+  text TEXT NOT NULL DEFAULT '',
+  timestamp INTEGER NOT NULL,
+  message_json TEXT,
+  PRIMARY KEY (chat_id, msg_id)
+);
+CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_msg ON whatsapp_messages(msg_id);
+CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_wa_id ON whatsapp_messages(wa_id);
+CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_time ON whatsapp_messages(timestamp);

package/src/storage/sql/statements.generated.ts

@@ -255,7 +255,28 @@ CREATE TABLE IF NOT EXISTS journal (
   type    TEXT    NOT NULL,
   payload TEXT    NOT NULL
 );
-CREATE INDEX IF NOT EXISTS idx_journal_type ON journal(type, seq);`;
+CREATE INDEX IF NOT EXISTS idx_journal_type ON journal(type, seq);
+
+-- WhatsApp message keys: Talon's numeric message id ↔ WhatsApp's
+-- (id, remoteJid, fromMe, participant) key, plus the full proto as JSON
+-- so react/reply/forward/download keep working on messages from before
+-- a restart and media past the CDN TTL can be re-requested.
+CREATE TABLE IF NOT EXISTS whatsapp_messages (
+  chat_id TEXT NOT NULL,
+  msg_id INTEGER NOT NULL,
+  wa_id TEXT NOT NULL,
+  remote_jid TEXT NOT NULL,
+  from_me INTEGER NOT NULL DEFAULT 0,
+  participant TEXT,
+  sender_name TEXT NOT NULL DEFAULT '',
+  text TEXT NOT NULL DEFAULT '',
+  timestamp INTEGER NOT NULL,
+  message_json TEXT,
+  PRIMARY KEY (chat_id, msg_id)
+);
+CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_msg ON whatsapp_messages(msg_id);
+CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_wa_id ON whatsapp_messages(wa_id);
+CREATE INDEX IF NOT EXISTS idx_whatsapp_messages_time ON whatsapp_messages(timestamp);`;
 
 export const chatSettingsSql = {
   upsert: `INSERT OR REPLACE INTO chat_settings (chat_id, settings) VALUES (?, ?)`,
@@ -361,6 +382,16 @@ SELECT msg_id, sender_id, sender_name, sender_handle, text, reply_to_msg_id,
 FROM history_messages
 WHERE chat_id = ?
   AND id IN (SELECT rowid FROM history_fts WHERE history_fts MATCH ?)
+ORDER BY id DESC LIMIT ?`,
+  searchFtsBetween: `-- searchFts restricted to a timestamp window [after, before) — the
+-- search_history \`after\` / \`before\` date parameters. Either bound may be
+-- the open end of the range (0 / a far-future value).
+SELECT msg_id, sender_id, sender_name, sender_handle, text, reply_to_msg_id,
+       timestamp, media_type, sticker_file_id, file_path
+FROM history_messages
+WHERE chat_id = ?
+  AND id IN (SELECT rowid FROM history_fts WHERE history_fts MATCH ?)
+  AND timestamp >= ? AND timestamp < ?
 ORDER BY id DESC LIMIT ?`,
   bySenderName: `-- The fragment param is LIKE-escaped by the repository (backslash escape).
 SELECT msg_id, sender_id, sender_name, sender_handle, text, reply_to_msg_id,
@@ -548,3 +579,21 @@ WHERE chat_id = ?1
     LIMIT ?2
   )`,
 } as const;
+
+export const whatsappMessagesSql = {
+  insert: `INSERT OR IGNORE INTO whatsapp_messages
+  (chat_id, msg_id, wa_id, remote_jid, from_me, participant, sender_name,
+   text, timestamp, message_json)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+  byMsgId: `SELECT chat_id, msg_id, wa_id, remote_jid, from_me, participant, sender_name,
+       text, timestamp, message_json
+FROM whatsapp_messages WHERE msg_id = ? LIMIT 1`,
+  byWaId: `-- Newest first: a WhatsApp id re-delivered on reconnect maps to the row
+-- that already exists for it.
+SELECT chat_id, msg_id, wa_id, remote_jid, from_me, participant, sender_name,
+       text, timestamp, message_json
+FROM whatsapp_messages WHERE wa_id = ? ORDER BY msg_id DESC LIMIT 1`,
+  maxMsgId: `SELECT MAX(msg_id) AS max_id FROM whatsapp_messages`,
+  deleteOlderThan: `DELETE FROM whatsapp_messages WHERE timestamp < ?`,
+  deleteAll: `DELETE FROM whatsapp_messages`,
+} as const;

package/src/storage/sql/whatsapp-messages.sql

@@ -0,0 +1,29 @@
+-- Statements for the whatsapp_messages table (see
+-- repositories/whatsapp-messages-repo.ts for parameter order and mapping).
+
+-- name: insert
+INSERT OR IGNORE INTO whatsapp_messages
+  (chat_id, msg_id, wa_id, remote_jid, from_me, participant, sender_name,
+   text, timestamp, message_json)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+
+-- name: byMsgId
+SELECT chat_id, msg_id, wa_id, remote_jid, from_me, participant, sender_name,
+       text, timestamp, message_json
+FROM whatsapp_messages WHERE msg_id = ? LIMIT 1
+
+-- name: byWaId
+-- Newest first: a WhatsApp id re-delivered on reconnect maps to the row
+-- that already exists for it.
+SELECT chat_id, msg_id, wa_id, remote_jid, from_me, participant, sender_name,
+       text, timestamp, message_json
+FROM whatsapp_messages WHERE wa_id = ? ORDER BY msg_id DESC LIMIT 1
+
+-- name: maxMsgId
+SELECT MAX(msg_id) AS max_id FROM whatsapp_messages
+
+-- name: deleteOlderThan
+DELETE FROM whatsapp_messages WHERE timestamp < ?
+
+-- name: deleteAll
+DELETE FROM whatsapp_messages

package/src/storage/whatsapp-messages.ts

@@ -0,0 +1,51 @@
+/**
+ * Persistent WhatsApp message keys.
+ *
+ * The WhatsApp frontend hands every message it sees a numeric Talon id
+ * and needs the WhatsApp key (and, for forwards and media re-downloads,
+ * the whole proto) back later. Its in-memory store is a bounded cache;
+ * this table is the record, so a reply, reaction or download addressed
+ * at a message from before the last restart still resolves.
+ *
+ * The proto travels as JSON — the frontend serialises and rehydrates it;
+ * storage never imports Baileys.
+ */
+
+import * as repo from "./repositories/whatsapp-messages-repo.js";
+
+export type { WhatsAppMessageRecord } from "./repositories/whatsapp-messages-repo.js";
+
+/** How long a message's key (and retained proto) stays retrievable. */
+const WHATSAPP_MESSAGE_RETENTION_MS = 180 * 24 * 60 * 60 * 1000;
+
+/** Persist a message; a (chat, msg_id) already stored is left as is. */
+export function saveWhatsAppMessage(record: repo.WhatsAppMessageRecord): void {
+  repo.insert(record);
+}
+
+export function getWhatsAppMessage(
+  msgId: number,
+): repo.WhatsAppMessageRecord | undefined {
+  return repo.byMsgId(msgId);
+}
+
+export function getWhatsAppMessageByWaId(
+  waId: string,
+): repo.WhatsAppMessageRecord | undefined {
+  return repo.byWaId(waId);
+}
+
+/** Highest Talon id ever persisted — the restart seed for the id counter. */
+export function maxWhatsAppMsgId(): number | undefined {
+  return repo.maxMsgId();
+}
+
+/** Drop rows past the retention window; returns the number removed. */
+export function pruneWhatsAppMessages(now = Date.now()): number {
+  return repo.deleteOlderThan(now - WHATSAPP_MESSAGE_RETENTION_MS);
+}
+
+/** Test seam: forget every persisted message. */
+export function clearWhatsAppMessages(): void {
+  repo.deleteAll();
+}