proton-mail-bridge-client 2.0.0 → 2.0.2

package/dist/services/local-index-service.js

@@ -2,6 +2,7 @@ import Database from "better-sqlite3";
 import { chmodSync } from "node:fs";
 import { mkdir, readFile, rm } from "node:fs/promises";
 import { dirname, join } from "node:path";
+import { ensureAccountIdentityMatches } from "../utils/account-identity.js";
 import { dedupeEmails, extractDomain, extractMessageIdList, lowerCaseAddress, nextDay, normalizeMailboxLabel, normalizeMessageId, normalizeSubjectForThread, sortEmailsByNewest, } from "../utils/helpers.js";
 import { logger } from "../utils/logger.js";
 const DB_SCHEMA_VERSION = 3;
@@ -401,8 +402,17 @@ export class LocalIndexService {
         return this.getStatus();
     }
     async getStatus() {
-        const snapshot = await this.loadSnapshot();
+        const db = await this.ensureDb();
+        const { ownerEmail, updatedAt, folders, indexedFolders } = this.loadFoldersAndMetadata(db);
+        const syncCheckpoints = this.loadCheckpointsSync(db);
         const counts = await this.realMessageCounts();
+        // threadCount/labelCount (below, via toStatus -> buildThreads/buildMailboxMessages)
+        // genuinely need message content — thread grouping via thread_id/References/In-Reply-To
+        // and per-message label normalization aren't derivable from folders/sync_state alone.
+        // Reuse the same deliberately-capped message query loadSnapshot() uses elsewhere for
+        // this; every other field above is now sourced without touching the messages table.
+        const messages = this.loadMessages(db, {});
+        const snapshot = { ownerEmail, updatedAt, folders, indexedFolders, syncCheckpoints, messages };
         return this.toStatus(snapshot, counts);
     }
     // loadSnapshot() caps snapshot.messages at DEFAULT_SNAPSHOT_LIMIT (5000) —
@@ -549,7 +559,28 @@ export class LocalIndexService {
             .slice(0, limit);
     }
     async getThreads(input = {}) {
-        const snapshot = await this.loadSnapshot({ folder: input.folder, label: input.label });
+        const db = await this.ensureDb();
+        const { ownerEmail, updatedAt, folders, indexedFolders } = this.loadFoldersAndMetadata(db);
+        const syncCheckpoints = this.loadCheckpointsSync(db);
+        let messages;
+        let messagesCapped = false;
+        const hasFilter = Boolean(input.folder || input.label || input.query);
+        if (hasFilter) {
+            // Filter/search in SQL first — mirroring how search() builds its SQL query
+            // before any snapshot-size limiting — so a thread whose messages sit entirely
+            // outside the DEFAULT_SNAPSHOT_LIMIT-capped snapshot is never silently missed
+            // just because a folder/label/query filter was given.
+            messages = this.loadThreadCandidateMessages(db, input);
+        }
+        else {
+            // No filter at all: a huge mailbox still needs SOME cap on what's fetched, so
+            // keep the deliberate DEFAULT_SNAPSHOT_LIMIT cap here — but surface it via
+            // messagesCapped instead of silently truncating, since real pagination over an
+            // un-scoped "every thread" view isn't possible without a separate thread index.
+            messages = this.loadMessages(db, {});
+            messagesCapped = messages.length >= DEFAULT_SNAPSHOT_LIMIT;
+        }
+        const snapshot = { ownerEmail, updatedAt, folders, indexedFolders, syncCheckpoints, messages };
         const threads = this.buildThreads(snapshot).filter((thread) => {
             if (input.label) {
                 const labelNeedle = input.label.toLowerCase();
@@ -579,9 +610,37 @@ export class LocalIndexService {
             hasMore: totalCount > offset + limit,
             threads: threads.slice(offset, offset + limit),
             ...this.indexFreshnessFields(snapshot.updatedAt),
+            ...(messagesCapped ? { messagesCapped: true } : {}),
         };
     }
     async getThreadById(threadId) {
+        const db = await this.ensureDb();
+        const imapPrefix = "imap:";
+        if (threadId.startsWith(imapPrefix)) {
+            const rawThreadId = threadId.slice(imapPrefix.length);
+            // A thread keyed as "imap:<id>" was grouped directly off the persisted
+            // messages.thread_id column (see threadKeyForEmail/resolveThreadKey below) —
+            // its full membership can be queried by that column directly, unbounded by
+            // the DEFAULT_SNAPSHOT_LIMIT cap. This is what lets a threadId that was valid
+            // when the index was smaller keep resolving after the index grows past 5000,
+            // instead of throwing "Thread not found" just because its messages aged out
+            // of the capped snapshot below.
+            const rows = db.prepare(`SELECT * FROM messages WHERE thread_id = ?`).all(rawThreadId);
+            if (rows.length > 0) {
+                const { ownerEmail, updatedAt, folders, indexedFolders } = this.loadFoldersAndMetadata(db);
+                const syncCheckpoints = this.loadCheckpointsSync(db);
+                const messages = rows.map((row) => this.rowToEmailSummary(row));
+                const snapshot = { ownerEmail, updatedAt, folders, indexedFolders, syncCheckpoints, messages };
+                const thread = this.buildThreads(snapshot, true).find((entry) => entry.id === threadId);
+                if (thread) {
+                    return thread;
+                }
+            }
+        }
+        // Threads keyed off a References/In-Reply-To chain ("ref:...") or the
+        // participant-signature fallback have no persisted thread_id to query by —
+        // resolving them needs the cross-message reference graph, so this path stays
+        // scoped to the capped snapshot as before.
         const snapshot = await this.loadSnapshot();
         const thread = this.buildThreads(snapshot, true).find((entry) => entry.id === threadId);
         if (!thread) {
@@ -654,8 +713,12 @@ export class LocalIndexService {
         };
     }
     async getSyncCheckpointMap() {
-        const snapshot = await this.loadSnapshot();
-        return Object.fromEntries(snapshot.syncCheckpoints.map((checkpoint) => [checkpoint.folder, checkpoint]));
+        // Query sync_state directly — no need to pay loadSnapshot()'s up-to-5000-row
+        // message deserialization just to read checkpoints, which never touch a
+        // message row at all.
+        const db = await this.ensureDb();
+        const checkpoints = this.loadCheckpointsSync(db);
+        return Object.fromEntries(checkpoints.map((checkpoint) => [checkpoint.folder, checkpoint]));
     }
     async getInboxDigest(input = {}) {
         const snapshot = await this.loadSnapshot();
@@ -890,6 +953,12 @@ export class LocalIndexService {
         if (this.db && this.initialized) {
             return this.db;
         }
+        // Refuse to open this dataDir's SQLite index if it belongs to a
+        // different account than the one currently configured (see
+        // account-identity.ts) — must run before the Database constructor below
+        // ever touches the file, so a mismatched account can never read a
+        // single row of the previous account's index.
+        await ensureAccountIdentityMatches(this.config.dataDir, this.config.smtp.username);
         await mkdir(dirname(this.dbPath), { recursive: true, mode: 0o700 });
         const isFirstOpen = !this.db;
         const db = this.db ?? new Database(this.dbPath);
@@ -1006,12 +1075,13 @@ export class LocalIndexService {
         attachments_json = excluded.attachments_json,
         attachment_text = COALESCE(excluded.attachment_text, messages.attachment_text),
         labels_json = excluded.labels_json
+      RETURNING preview, attachment_text
     `);
         const upsertSyncState = db.prepare(`
       INSERT INTO sync_state (
-        folder, uid_validity, uid_next, highest_uid, last_sync_at, last_full_sync_at, strategy, changed, fetched, total, backfilled_to_uid
+        folder, uid_validity, uid_next, highest_uid, last_sync_at, last_full_sync_at, strategy, changed, fetched, total, backfilled_to_uid, incremental_resume_uid
       ) VALUES (
-        @folder, @uid_validity, @uid_next, @highest_uid, @last_sync_at, @last_full_sync_at, @strategy, @changed, @fetched, @total, @backfilled_to_uid
+        @folder, @uid_validity, @uid_next, @highest_uid, @last_sync_at, @last_full_sync_at, @strategy, @changed, @fetched, @total, @backfilled_to_uid, @incremental_resume_uid
       )
       ON CONFLICT(folder) DO UPDATE SET
         uid_validity = excluded.uid_validity,
@@ -1023,7 +1093,8 @@ export class LocalIndexService {
         changed = excluded.changed,
         fetched = excluded.fetched,
         total = excluded.total,
-        backfilled_to_uid = excluded.backfilled_to_uid
+        backfilled_to_uid = excluded.backfilled_to_uid,
+        incremental_resume_uid = excluded.incremental_resume_uid
     `);
         const deleteFts = db.prepare(`DELETE FROM messages_fts WHERE email_id = ?`);
         const insertFts = db.prepare(`
@@ -1066,6 +1137,19 @@ export class LocalIndexService {
                     foldersWithResetCheckpoint.add(folderStat.folder);
                 }
             }
+            // The "empty" strategy carries no UID range at all, so the range-scoped
+            // expunge-detection below (cleanupExpunged) never runs for it — a folder
+            // genuinely emptied on the server would otherwise keep its previously-
+            // indexed messages forever. folderObservedEmpty is only set when the
+            // server reported exists === 0 on a successful SELECT, never on a
+            // failed/interrupted fetch, so this purge only fires on a real
+            // observation. Scoped strictly to that one folder.
+            for (const folderStat of input.folderStats) {
+                if (folderStat.folderObservedEmpty) {
+                    deleteFtsForFolder.run(folderStat.folder);
+                    deleteMessagesForFolder.run(folderStat.folder);
+                }
+            }
             for (const folder of input.folders) {
                 const folderStat = input.folderStats.find((entry) => entry.folder === folder.path);
                 upsertFolder.run({
@@ -1099,10 +1183,18 @@ export class LocalIndexService {
                     fetched: folderStat.fetched ?? null,
                     total: folderStat.total ?? null,
                     backfilled_to_uid: resetCheckpoint ? null : folderStat.backfilledToUid ?? null,
+                    incremental_resume_uid: resetCheckpoint ? null : folderStat.incrementalResumeUid ?? null,
                 });
             }
             for (const email of input.emails) {
-                upsertMessage.run({
+                // RETURNING gives back the post-COALESCE stored values, not the raw
+                // incoming ones — a flags-only sync omits source/preview, and the
+                // messages table upsert already preserves the prior indexed preview/
+                // attachment_text in that case (IMAP content for a fixed UID is
+                // immutable). Without this, the FTS row below would be rebuilt from
+                // the incoming (empty) values and lose body-text searchability on
+                // every metadata-only refresh, even though the stored row is intact.
+                const persisted = upsertMessage.get({
                     email_id: email.id,
                     folder: email.folder,
                     uid: email.uid,
@@ -1129,9 +1221,10 @@ export class LocalIndexService {
                     attachment_text: email.attachmentText ?? null,
                     labels_json: JSON.stringify(email.labels),
                 });
-                const search = emailToSearchParts(email);
+                const mergedPreview = persisted.preview ?? "";
+                const search = emailToSearchParts({ ...email, attachmentText: persisted.attachment_text ?? undefined });
                 deleteFts.run(email.id);
-                insertFts.run(email.id, email.subject, email.preview ?? "", email.folder, search.labels, search.participants, search.attachmentNames);
+                insertFts.run(email.id, email.subject, mergedPreview, email.folder, search.labels, search.participants, search.attachmentNames);
             }
             if (cleanupExpunged) {
                 createSnapshotUidTable.run();
@@ -1259,7 +1352,8 @@ export class LocalIndexService {
         changed INTEGER NOT NULL DEFAULT 0,
         fetched INTEGER,
         total INTEGER,
-        backfilled_to_uid INTEGER
+        backfilled_to_uid INTEGER,
+        incremental_resume_uid INTEGER
       );
 
       CREATE INDEX IF NOT EXISTS idx_messages_folder ON messages(folder);
@@ -1294,6 +1388,9 @@ export class LocalIndexService {
         if (!syncStateColumns.has("backfilled_to_uid")) {
             db.exec(`ALTER TABLE sync_state ADD COLUMN backfilled_to_uid INTEGER`);
         }
+        if (!syncStateColumns.has("incremental_resume_uid")) {
+            db.exec(`ALTER TABLE sync_state ADD COLUMN incremental_resume_uid INTEGER`);
+        }
     }
     async maybeImportLegacyIndex(db) {
         const hasMetadata = db.prepare(`SELECT value FROM metadata WHERE key = 'schemaVersion'`).get();
@@ -1460,8 +1557,10 @@ export class LocalIndexService {
             };
         }
     }
-    async loadSnapshot(options = {}) {
-        const db = await this.ensureDb();
+    // Metadata/folders only — no messages table query at all. Split out of loadSnapshot()
+    // so callers that don't need message rows (getStatus, getThreads, getThreadById) don't
+    // pay for them.
+    loadFoldersAndMetadata(db) {
         const metadataRows = db
             .prepare(`SELECT key, value FROM metadata`)
             .all();
@@ -1485,9 +1584,20 @@ export class LocalIndexService {
             lastIndexedAt: row.last_indexed_at,
             lastIndexedCount: row.last_indexed_count,
         }));
-        const syncCheckpoints = db
+        return {
+            ownerEmail: metadata.ownerEmail || undefined,
+            updatedAt: metadata.updatedAt || undefined,
+            folders,
+            indexedFolders,
+        };
+    }
+    // sync_state only — no messages table query. Used directly by getSyncCheckpointMap()
+    // (which never needs a message row) and by loadSnapshot()/getStatus()/getThreads()/
+    // getThreadById() so none of them duplicate this query inline.
+    loadCheckpointsSync(db) {
+        return db
             .prepare(`
-        SELECT folder, uid_validity, uid_next, highest_uid, last_sync_at, last_full_sync_at, strategy, changed, fetched, total, backfilled_to_uid
+        SELECT folder, uid_validity, uid_next, highest_uid, last_sync_at, last_full_sync_at, strategy, changed, fetched, total, backfilled_to_uid, incremental_resume_uid
         FROM sync_state
         ORDER BY folder ASC
       `)
@@ -1511,7 +1621,18 @@ export class LocalIndexService {
             // the first sync_emails(full:true) call after upgrading returned
             // changed:false, fetched:0 instead of starting the newest window.
             backfilledToUid: row.backfilled_to_uid ?? undefined,
+            // Same NULL-vs-undefined pitfall as backfilledToUid above: planFolderSync
+            // treats undefined as "no incremental catch-up in progress" and a real
+            // number as "resume from here" — a null read back as null (not undefined)
+            // would be indistinguishable from a legitimate resume-at-0 edge case in
+            // some comparisons, so map it away explicitly here too.
+            incrementalResumeUid: row.incremental_resume_uid ?? undefined,
         }));
+    }
+    // The message-row query, capped at DEFAULT_SNAPSHOT_LIMIT by default — split out of
+    // loadSnapshot() so getThreads() can call it directly (or substitute
+    // loadThreadCandidateMessages() below) without re-fetching folders/checkpoints.
+    loadMessages(db, options) {
         const messageSqlParts = [`SELECT * FROM messages`];
         const messageParams = [];
         const messageConditions = [];
@@ -1540,13 +1661,65 @@ export class LocalIndexService {
         const messageOffset = Math.max(0, options.offset ?? 0);
         messageSqlParts.push(`ORDER BY COALESCE(internal_date, date) DESC, uid DESC LIMIT ? OFFSET ?`);
         messageParams.push(messageLimit, messageOffset);
-        const messages = db
+        return db
             .prepare(messageSqlParts.join(" "))
             .all(...messageParams)
             .map((row) => this.rowToEmailSummary(row));
+    }
+    // getThreads()'s filtered path: filter/search in SQL first (the same pattern
+    // loadCandidateEmails()/search() already use — a safe LIKE-based superset over every
+    // field the JS-level filter in getThreads() checks), then expand each match's
+    // persisted thread_id membership directly by that column, unbounded by
+    // DEFAULT_SNAPSHOT_LIMIT — so a thread is found and built complete even when most of
+    // its messages sit outside the newest 5000.
+    loadThreadCandidateMessages(db, input) {
+        const conditions = [];
+        const params = [];
+        if (input.folder) {
+            conditions.push(`folder = ?`);
+            params.push(input.folder);
+        }
+        if (input.label) {
+            const labelNeedle = escapeLike(input.label.toLowerCase());
+            conditions.push(`(LOWER(folder) LIKE ? ESCAPE '\\' OR LOWER(labels_json) LIKE ? ESCAPE '\\')`);
+            params.push(`%${labelNeedle}%`, `%${labelNeedle}%`);
+        }
+        if (input.query) {
+            const needle = `%${escapeLike(input.query.toLowerCase())}%`;
+            conditions.push(`(LOWER(subject) LIKE ? ESCAPE '\\' OR LOWER(from_json) LIKE ? ESCAPE '\\' OR LOWER(to_json) LIKE ? ESCAPE '\\' OR LOWER(cc_json) LIKE ? ESCAPE '\\' OR LOWER(bcc_json) LIKE ? ESCAPE '\\' OR LOWER(labels_json) LIKE ? ESCAPE '\\')`);
+            params.push(needle, needle, needle, needle, needle, needle);
+        }
+        const sql = `SELECT * FROM messages${conditions.length > 0 ? ` WHERE ${conditions.join(" AND ")}` : ""}`;
+        const candidates = db
+            .prepare(sql)
+            .all(...params)
+            .map((row) => this.rowToEmailSummary(row));
+        const threadIds = [...new Set(candidates.map((email) => email.threadId?.trim()).filter((id) => Boolean(id)))];
+        if (threadIds.length === 0) {
+            return candidates;
+        }
+        const byId = new Map(candidates.map((email) => [email.id, email]));
+        const CHUNK_SIZE = 200;
+        for (let i = 0; i < threadIds.length; i += CHUNK_SIZE) {
+            const chunk = threadIds.slice(i, i + CHUNK_SIZE);
+            const rows = db
+                .prepare(`SELECT * FROM messages WHERE thread_id IN (${chunk.map(() => "?").join(", ")})`)
+                .all(...chunk);
+            for (const row of rows) {
+                const email = this.rowToEmailSummary(row);
+                byId.set(email.id, email);
+            }
+        }
+        return [...byId.values()];
+    }
+    async loadSnapshot(options = {}) {
+        const db = await this.ensureDb();
+        const { ownerEmail, updatedAt, folders, indexedFolders } = this.loadFoldersAndMetadata(db);
+        const syncCheckpoints = this.loadCheckpointsSync(db);
+        const messages = this.loadMessages(db, options);
         return {
-            ownerEmail: metadata.ownerEmail || undefined,
-            updatedAt: metadata.updatedAt || undefined,
+            ownerEmail,
+            updatedAt,
             folders,
             indexedFolders,
             syncCheckpoints,