talon-agent 1.0.0

package/src/storage/history.ts

@@ -0,0 +1,278 @@
+/**
+ * Group message history buffer. Stores recent messages from all users
+ * so Claude has full conversation context even for messages that didn't
+ * trigger the bot.
+ *
+ * Persisted to disk — survives restarts so pulse, search, and group
+ * threading context don't lose state.
+ */
+
+import { existsSync, readFileSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import writeFileAtomic from "write-file-atomic";
+import { log, logError } from "../util/log.js";
+import { files } from "../util/paths.js";
+import { formatSmartTimestamp } from "../util/time.js";
+
+export type HistoryMessage = {
+  msgId: number;
+  senderId: number;
+  senderName: string;
+  text: string;
+  replyToMsgId?: number;
+  timestamp: number;
+  mediaType?:
+    | "photo"
+    | "document"
+    | "voice"
+    | "sticker"
+    | "video"
+    | "animation";
+  stickerFileId?: string;
+  /** Saved file path for downloaded media. */
+  filePath?: string;
+};
+
+const MAX_HISTORY_PER_CHAT = 500;
+const MAX_CHAT_COUNT = 1000;
+const STORE_FILE = files.history;
+
+const chatHistories = new Map<string, HistoryMessage[]>();
+let dirty = false;
+
+// ── Persistence ─────────────────────────────────────────────────────────────
+
+export function loadHistory(): void {
+  try {
+    if (existsSync(STORE_FILE)) {
+      const raw = JSON.parse(readFileSync(STORE_FILE, "utf-8")) as Record<
+        string,
+        HistoryMessage[]
+      >;
+      for (const [chatId, messages] of Object.entries(raw)) {
+        // Only load recent messages (cap per chat)
+        const trimmed = messages.slice(-MAX_HISTORY_PER_CHAT);
+        chatHistories.set(chatId, trimmed);
+      }
+      log("sessions", `Loaded history for ${chatHistories.size} chat(s)`);
+    }
+  } catch {
+    // Primary file corrupt — try backup
+    const bakFile = STORE_FILE + ".bak";
+    try {
+      if (existsSync(bakFile)) {
+        const raw = JSON.parse(readFileSync(bakFile, "utf-8")) as Record<string, HistoryMessage[]>;
+        for (const [chatId, messages] of Object.entries(raw)) {
+          chatHistories.set(chatId, messages.slice(-MAX_HISTORY_PER_CHAT));
+        }
+        logError("sessions", "Loaded history from backup (primary was corrupt)");
+        return;
+      }
+    } catch { /* backup also corrupt */ }
+    logError("sessions", "History data corrupt and no valid backup — starting fresh");
+  }
+}
+
+function saveHistory(): void {
+  if (!dirty) return;
+  try {
+    const dir = dirname(STORE_FILE);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    const obj: Record<string, HistoryMessage[]> = {};
+    for (const [chatId, messages] of chatHistories) {
+      obj[chatId] = messages;
+    }
+    const data = JSON.stringify(obj) + "\n";
+    // Write backup of current file before overwriting
+    if (existsSync(STORE_FILE)) {
+      try { writeFileAtomic.sync(STORE_FILE + ".bak", readFileSync(STORE_FILE)); } catch { /* best effort */ }
+    }
+    writeFileAtomic.sync(STORE_FILE, data);
+    dirty = false;
+  } catch (err) {
+    logError("sessions", "Failed to persist history", err);
+  }
+}
+
+// Auto-save every 30 seconds (less frequent than sessions since history is larger)
+const autoSaveTimer = setInterval(saveHistory, 30_000);
+process.on("exit", saveHistory);
+
+export function flushHistory(): void {
+  clearInterval(autoSaveTimer);
+  dirty = true;
+  saveHistory();
+}
+
+// ── Core operations ─────────────────────────────────────────────────────────
+
+export function pushMessage(chatId: string, msg: HistoryMessage): void {
+  let history = chatHistories.get(chatId);
+  if (!history) {
+    if (chatHistories.size >= MAX_CHAT_COUNT) {
+      const evictCount = Math.floor(MAX_CHAT_COUNT * 0.1);
+      const iter = chatHistories.keys();
+      for (let i = 0; i < evictCount; i++) {
+        const oldest = iter.next();
+        if (oldest.done) break;
+        chatHistories.delete(oldest.value);
+      }
+      // Mark dirty so evicted chats are removed from disk on next save
+      dirty = true;
+    }
+    history = [];
+    chatHistories.set(chatId, history);
+  }
+  history.push(msg);
+  if (history.length > MAX_HISTORY_PER_CHAT) {
+    history.splice(0, history.length - MAX_HISTORY_PER_CHAT);
+  }
+  dirty = true;
+}
+
+export function getRecentHistory(chatId: string, limit = 50): HistoryMessage[] {
+  const history = chatHistories.get(chatId);
+  if (!history) return [];
+  return history.slice(-limit);
+}
+
+/** Update a message's file path after media download. */
+export function setMessageFilePath(chatId: string, msgId: number, filePath: string): void {
+  const history = chatHistories.get(chatId);
+  if (!history) return;
+  const msg = history.find((m) => m.msgId === msgId);
+  if (msg) { msg.filePath = filePath; dirty = true; }
+}
+
+export function clearHistory(chatId: string): void {
+  chatHistories.delete(chatId);
+  dirty = true;
+}
+
+// ── Formatted queries ───────────────────────────────────────────────────────
+
+function formatMessage(m: HistoryMessage): string {
+  const replyTag = m.replyToMsgId ? ` (replying to msg:${m.replyToMsgId})` : "";
+  const mediaTag = m.mediaType ? ` [${m.mediaType}]` : "";
+  const stickerTag = m.stickerFileId
+    ? ` (sticker_file_id: ${m.stickerFileId})`
+    : "";
+  const fileTag = m.filePath ? ` (file: ${m.filePath})` : "";
+  const time = formatSmartTimestamp(m.timestamp);
+  return `[msg:${m.msgId} ${time}] ${m.senderName}${replyTag}${mediaTag}${stickerTag}${fileTag}: ${m.text}`;
+}
+
+export function getRecentFormatted(chatId: string, limit = 20): string {
+  const messages = getRecentHistory(chatId, limit);
+  if (messages.length === 0) return "No messages in history.";
+  return messages.map(formatMessage).join("\n");
+}
+
+export function searchHistory(
+  chatId: string,
+  query: string,
+  limit = 20,
+): string {
+  const history = chatHistories.get(chatId);
+  if (!history || history.length === 0) return "No messages in history.";
+  const lower = query.toLowerCase();
+  const matches = history.filter(
+    (m) =>
+      m.text.toLowerCase().includes(lower) ||
+      m.senderName.toLowerCase().includes(lower),
+  );
+  if (matches.length === 0) return `No messages matching "${query}".`;
+  return matches.slice(-limit).map(formatMessage).join("\n");
+}
+
+export function getMessagesByUser(
+  chatId: string,
+  userName: string,
+  limit = 20,
+): string {
+  const history = chatHistories.get(chatId);
+  if (!history || history.length === 0) return "No messages in history.";
+  const lower = userName.toLowerCase();
+  const matches = history.filter((m) =>
+    m.senderName.toLowerCase().includes(lower),
+  );
+  if (matches.length === 0) return `No messages from "${userName}".`;
+  return matches.slice(-limit).map(formatMessage).join("\n");
+}
+
+export function getMessageById(chatId: string, msgId: number): string {
+  const history = chatHistories.get(chatId);
+  if (!history) return "No messages in history.";
+  const msg = history.find((m) => m.msgId === msgId);
+  if (!msg) return `Message ${msgId} not found in recent history.`;
+  return formatMessage(msg);
+}
+
+export function getKnownUsers(chatId: string): string {
+  const history = chatHistories.get(chatId);
+  if (!history || history.length === 0) return "No users seen yet.";
+  const users = new Map<
+    number,
+    { name: string; lastSeen: number; messageCount: number }
+  >();
+  for (const m of history) {
+    const existing = users.get(m.senderId);
+    if (!existing || m.timestamp > existing.lastSeen) {
+      users.set(m.senderId, {
+        name: m.senderName,
+        lastSeen: m.timestamp,
+        messageCount: (existing?.messageCount ?? 0) + 1,
+      });
+    } else {
+      existing.messageCount++;
+    }
+  }
+  const lines = [...users.entries()]
+    .sort((a, b) => b[1].lastSeen - a[1].lastSeen)
+    .map(([id, u]) => {
+      const ago = formatTimeAgo(u.lastSeen);
+      return `${u.name} (user_id: ${id}) — ${u.messageCount} msgs, last seen ${ago}`;
+    });
+  return lines.join("\n");
+}
+
+function formatTimeAgo(ts: number): string {
+  const diff = Date.now() - ts;
+  if (diff < 60_000) return "just now";
+  if (diff < 3_600_000) return `${Math.floor(diff / 60_000)}m ago`;
+  if (diff < 86_400_000) return `${Math.floor(diff / 3_600_000)}h ago`;
+  return `${Math.floor(diff / 86_400_000)}d ago`;
+}
+
+export function getRecentBySenderId(
+  chatId: string,
+  senderId: number,
+  limit = 5,
+): HistoryMessage[] {
+  const history = chatHistories.get(chatId);
+  if (!history) return [];
+  const matches = history.filter((m) => m.senderId === senderId);
+  return matches.slice(-limit);
+}
+
+export function getLatestMessageId(chatId: string): number | undefined {
+  const history = chatHistories.get(chatId);
+  if (!history || history.length === 0) return undefined;
+  return history[history.length - 1].msgId;
+}
+
+export function getHistoryStats(chatId: string): {
+  totalMessages: number;
+  uniqueUsers: number;
+  oldestTimestamp: number;
+  newestTimestamp: number;
+} {
+  const history = chatHistories.get(chatId) ?? [];
+  const users = new Set(history.map((m) => m.senderId));
+  return {
+    totalMessages: history.length,
+    uniqueUsers: users.size,
+    oldestTimestamp: history[0]?.timestamp ?? 0,
+    newestTimestamp: history[history.length - 1]?.timestamp ?? 0,
+  };
+}

package/src/storage/media-index.ts

@@ -0,0 +1,116 @@
+/**
+ * Media index — tracks downloaded media files with metadata and expiry.
+ *
+ * Provides fast lookup of recent photos/files by chat, sender, or type.
+ * Auto-expires entries older than RETENTION_DAYS.
+ * Persisted to .talon/data/media-index.json.
+ */
+
+import { existsSync, readFileSync, mkdirSync, unlinkSync } from "node:fs";
+import { dirname } from "node:path";
+import writeFileAtomic from "write-file-atomic";
+import { log } from "../util/log.js";
+import { files } from "../util/paths.js";
+
+export type MediaEntry = {
+  id: string;           // unique key: chatId:msgId
+  chatId: string;
+  msgId: number;
+  senderName: string;
+  type: "photo" | "document" | "voice" | "video" | "animation" | "audio" | "sticker";
+  filePath: string;
+  caption?: string;
+  timestamp: number;
+};
+
+const STORE_FILE = files.mediaIndex;
+const RETENTION_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+
+let entries: MediaEntry[] = [];
+let dirty = false;
+
+// ── Persistence ─────────────────────────────────────────────────────────────
+
+export function loadMediaIndex(): void {
+  try {
+    if (existsSync(STORE_FILE)) {
+      entries = JSON.parse(readFileSync(STORE_FILE, "utf-8"));
+    }
+  } catch { entries = []; }
+  // Purge expired on load
+  purgeExpired();
+}
+
+function save(): void {
+  if (!dirty) return;
+  try {
+    const dir = dirname(STORE_FILE);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    writeFileAtomic.sync(STORE_FILE, JSON.stringify(entries) + "\n");
+    dirty = false;
+  } catch { /* non-fatal */ }
+}
+
+const autoSaveTimer = setInterval(save, 30_000);
+process.on("exit", save);
+
+export function flushMediaIndex(): void {
+  clearInterval(autoSaveTimer);
+  dirty = true;
+  save();
+}
+
+// ── CRUD ────────────────────────────────────────────────────────────────────
+
+export function addMedia(entry: Omit<MediaEntry, "id">): void {
+  const id = `${entry.chatId}:${entry.msgId}`;
+  // Dedupe
+  const existing = entries.findIndex((e) => e.id === id);
+  if (existing >= 0) entries[existing] = { ...entry, id };
+  else entries.push({ ...entry, id });
+  dirty = true;
+}
+
+/** Get recent media for a chat, newest first. */
+export function getRecentMedia(chatId: string, limit = 10): MediaEntry[] {
+  return entries
+    .filter((e) => e.chatId === chatId)
+    .sort((a, b) => b.timestamp - a.timestamp)
+    .slice(0, limit);
+}
+
+/** Get all media matching a type in a chat. */
+export function getMediaByType(chatId: string, type: MediaEntry["type"], limit = 10): MediaEntry[] {
+  return entries
+    .filter((e) => e.chatId === chatId && e.type === type)
+    .sort((a, b) => b.timestamp - a.timestamp)
+    .slice(0, limit);
+}
+
+/** Format media index as text for Claude. */
+export function formatMediaIndex(chatId: string, limit = 10): string {
+  const media = getRecentMedia(chatId, limit);
+  if (media.length === 0) return "No recent media in this chat.";
+  return media.map((m) => {
+    const time = new Date(m.timestamp).toISOString().slice(0, 16).replace("T", " ");
+    const cap = m.caption ? ` "${m.caption.slice(0, 50)}"` : "";
+    return `[${m.type}] msg:${m.msgId} by ${m.senderName} at ${time}${cap}\n  file: ${m.filePath}`;
+  }).join("\n");
+}
+
+// ── Expiry ──────────────────────────────────────────────────────────────────
+
+function purgeExpired(): void {
+  const cutoff = Date.now() - RETENTION_MS;
+  const before = entries.length;
+  entries = entries.filter((e) => {
+    if (e.timestamp >= cutoff) return true;
+    // Delete the file too
+    try { if (existsSync(e.filePath)) unlinkSync(e.filePath); } catch { /* skip */ }
+    return false;
+  });
+  if (entries.length < before) {
+    dirty = true;
+    log("workspace", `Purged ${before - entries.length} expired media entries`);
+  }
+}