stellar-drive 1.2.14 → 1.2.16

package/dist/engine.js

@@ -138,6 +138,26 @@ function getColumns(name) {
     const table = getEngineConfig().tables.find((t) => t.supabaseName === name || t.schemaKey === name);
     return table?.columns || '*';
 }
+/**
+ * Guarantee mandatory system-field defaults on an outbound sync payload.
+ *
+ * Every synced table in Supabase has `deleted boolean not null default false`.
+ * When an entity was written to IndexedDB by older code paths (before the
+ * `deleted` column existed) OR by a caller that passed a create payload
+ * without explicitly setting `deleted`, the field serializes as `undefined`
+ * (dropped by JSON) or `null` — both of which violate the NOT NULL
+ * constraint server-side.
+ *
+ * Rather than scattering this fix across every payload-assembly site, call
+ * this helper right before `filterPayloadToSchema()` so no code path can
+ * forget. Mutates and returns the same object for convenience.
+ */
+function ensureSystemFieldDefaults(payload) {
+    if (payload.deleted === undefined || payload.deleted === null) {
+        payload.deleted = false;
+    }
+    return payload;
+}
 /**
  * Filter a payload to only include columns defined in the schema.
  *
@@ -197,7 +217,10 @@ function getTombstoneMaxAgeDays() {
 }
 /** Minimum time tab must be hidden before triggering a sync on return. Default: 5min */
 function getVisibilitySyncMinAwayMs() {
-    return getEngineConfig().visibilitySyncMinAwayMs ?? 300000;
+    // 15 min default. No data loss: local writes always push immediately, and realtime
+    // (restarted on tab visible by Phase 1) delivers remote changes in real-time.
+    // This threshold only controls when a *poll* runs as backup on return.
+    return getEngineConfig().visibilitySyncMinAwayMs ?? 900000;
 }
 /** Cooldown after a successful sync before allowing reconnect-triggered sync. Default: 2min */
 function getOnlineReconnectCooldownMs() {
@@ -412,6 +435,20 @@ let isTabVisible = true;
 let visibilityDebounceTimeout = null;
 /** When the tab became hidden (null if currently visible) — used to calculate away duration */
 let tabHiddenAt = null;
+/** Timer for tearing down realtime after tab is hidden for 15s */
+let realtimeTeardownTimer = null;
+/**
+ * Delay before tearing down realtime when tab is hidden.
+ * 15s prevents thrashing on quick tab switches (checking a notification and
+ * returning often takes 8-12s). After this delay, the WebSocket channel is
+ * fully closed — eliminating heartbeat egress (~21 pings/min) while hidden.
+ * Remote changes are caught by the visibility sync poll on return.
+ */
+const REALTIME_TEARDOWN_DELAY_MS = 15000;
+/** Whether realtime was torn down while tab was hidden (needs restart on return) */
+let realtimeWasTornDown = false;
+/** When realtime was last (re)started — used for reconnect grace period in periodic sync */
+let lastRealtimeStartAt = 0;
 /** Debounce delay for visibility-change syncs (prevents rapid tab-switching spam) */
 const VISIBILITY_SYNC_DEBOUNCE_MS = 1000;
 /**
@@ -523,8 +560,13 @@ let lockPromise = null;
 let lockResolve = null;
 /** Timestamp when the lock was acquired (for stale-lock detection) */
 let lockAcquiredAt = null;
-/** Maximum time a sync lock can be held before force-release (60 seconds) */
-const SYNC_LOCK_TIMEOUT_MS = 60000;
+/**
+ * Maximum time a sync lock can be held before force-release.
+ * 6 minutes — exceeds the max operation timeout (5 min push/pull) by 1 min
+ * headroom, preventing the watchdog from killing legitimate large batch
+ * operations (e.g. 10,000+ row push = 20 batch upserts of 500).
+ */
+const SYNC_LOCK_TIMEOUT_MS = 360000;
 // --- Event listener references (stored for cleanup in stopSyncEngine) ---
 let handleOnlineRef = null;
 let handleOfflineRef = null;
@@ -941,13 +983,14 @@ async function pullRemoteChanges(minCursor) {
         let offset = 0;
         let hasMore = true;
         while (hasMore) {
-            const { data, error } = await supabase
+            const { data, error } = (await withTimeout(Promise.resolve(supabase
                 .from(table.supabaseName)
                 .select(table.columns)
                 .gt('updated_at', lastSync)
                 .order('updated_at', { ascending: true })
                 .order('id', { ascending: true })
-                .range(offset, offset + PULL_PAGE_SIZE - 1);
+                .range(offset, offset + PULL_PAGE_SIZE - 1)), 30000, `Pull ${table.supabaseName} page at offset ${offset}`));
+            debugLog(`[SYNC] Pulled ${table.supabaseName} offset=${offset} rows=${data?.length ?? 0}`);
             if (error)
                 return { data: allData, error };
             if (!data)
@@ -958,8 +1001,10 @@ async function pullRemoteChanges(minCursor) {
         }
         return { data: allData, error: null };
     }
-    // Wrapped in timeout to prevent hanging if Supabase doesn't respond
-    const results = await withTimeout(Promise.all(config.tables.map(pullTablePaginated)), 30000, 'Pull remote changes');
+    // No global timeout here — outer timeout in runFullSync wraps the entire pull.
+    // Per-page timeouts (below) catch individual hung requests without killing
+    // large pulls (10,000+ rows = many sequential pages).
+    const results = await Promise.all(config.tables.map(pullTablePaginated));
     // Check for errors
     for (let i = 0; i < results.length; i++) {
         if (results[i].error)
@@ -1145,148 +1190,322 @@ async function pushPendingOps() {
     // "pending" state between sync cycles instead of silently consuming them.
     const snapshotItems = await getPendingSync();
     const snapshotIds = new Set(snapshotItems.map((item) => item.id));
-    while (iterations < maxIterations) {
-        const pendingItems = (await getPendingSync()).filter((item) => snapshotIds.has(item.id));
-        if (pendingItems.length === 0)
-            break;
-        iterations++;
-        let processedAny = false;
-        // ── Batch creates: group by table and INSERT in bulk ──
-        // This is critical for performance: CSV imports with hundreds of transactions
-        // push in a few batch calls instead of hundreds of individual HTTP requests.
-        const createItems = pendingItems.filter((item) => item.operationType === 'create');
-        const nonCreateItems = pendingItems.filter((item) => item.operationType !== 'create');
-        if (createItems.length > 0) {
-            // Group creates by table, preserving queue order within each group.
-            // Bulk-read all sync queue IDs in one IndexedDB call instead of
-            // per-item reads (N sequential reads → 1 bulk read).
-            const createQueueIds = createItems.filter((item) => item.id).map((item) => item.id);
-            const queuedRows = await db.table('syncQueue').bulkGet(createQueueIds);
-            const stillQueuedIds = new Set(queuedRows
-                .map((row, i) => (row ? createQueueIds[i] : null))
-                .filter((id) => id !== null));
-            const createsByTable = new Map();
-            for (const item of createItems) {
-                if (item.id && !stillQueuedIds.has(item.id))
-                    continue;
-                const existing = createsByTable.get(item.table) || [];
-                existing.push(item);
-                createsByTable.set(item.table, existing);
-            }
-            // Sort table order: parent tables before child tables to satisfy RLS FK checks.
-            const schema = getEngineConfig().schema;
-            const sortedTableEntries = [...createsByTable.entries()].sort(([tableA], [tableB]) => {
-                if (!schema)
+    // Start progress tracking for high-volume pushes so the UI can show
+    // "1,200 of 2,500 changes synced…" instead of an opaque spinner that the
+    // user interprets as "stuck". Threshold matches the realtime-suspend
+    // threshold so the two cinematic "heavy sync" signals are in lockstep.
+    //
+    // A periodic monitor reads the live IndexedDB queue size every 400ms and
+    // publishes it through the syncStatusStore. Polling IndexedDB is cheap
+    // (indexed count query on a small table) and keeps the UI accurate no
+    // matter which internal push path (batch, per-item fallback, singleton,
+    // coalesced, duplicate-retry) the items take — we don't have to sprinkle
+    // progress calls across every branch.
+    const PROGRESS_THRESHOLD = 50;
+    const trackProgress = snapshotItems.length >= PROGRESS_THRESHOLD;
+    let progressMonitor = null;
+    let lastReportedCompleted = 0;
+    if (trackProgress) {
+        syncStatusStore.startProgress(snapshotItems.length);
+        syncStatusStore.setPendingCount(snapshotItems.length);
+        progressMonitor = setInterval(async () => {
+            try {
+                const liveCount = await db.table('syncQueue').count();
+                const completed = Math.max(0, snapshotItems.length - liveCount);
+                syncStatusStore.setPendingCount(liveCount);
+                const delta = completed - lastReportedCompleted;
+                if (delta !== 0) {
+                    syncStatusStore.advanceProgress(delta);
+                    lastReportedCompleted = completed;
+                }
+            }
+            catch (err) {
+                debugWarn('[SYNC] Progress monitor tick failed:', err);
+            }
+        }, 400);
+    }
+    try {
+        while (iterations < maxIterations) {
+            const pendingItems = (await getPendingSync()).filter((item) => snapshotIds.has(item.id));
+            if (pendingItems.length === 0)
+                break;
+            iterations++;
+            let processedAny = false;
+            // ── Batch creates: group by table and INSERT in bulk ──
+            // This is critical for performance: CSV imports with hundreds of transactions
+            // push in a few batch calls instead of hundreds of individual HTTP requests.
+            const createItems = pendingItems.filter((item) => item.operationType === 'create');
+            const nonCreateItems = pendingItems.filter((item) => item.operationType !== 'create');
+            if (createItems.length > 0) {
+                // Group creates by table, preserving queue order within each group.
+                // Bulk-read all sync queue IDs in one IndexedDB call instead of
+                // per-item reads (N sequential reads → 1 bulk read).
+                const createQueueIds = createItems.filter((item) => item.id).map((item) => item.id);
+                const queuedRows = await db.table('syncQueue').bulkGet(createQueueIds);
+                const stillQueuedIds = new Set(queuedRows
+                    .map((row, i) => (row ? createQueueIds[i] : null))
+                    .filter((id) => id !== null));
+                const createsByTable = new Map();
+                for (const item of createItems) {
+                    if (item.id && !stillQueuedIds.has(item.id))
+                        continue;
+                    const existing = createsByTable.get(item.table) || [];
+                    existing.push(item);
+                    createsByTable.set(item.table, existing);
+                }
+                // Sort table order: parent tables before child tables to satisfy RLS FK checks.
+                const schema = getEngineConfig().schema;
+                const sortedTableEntries = [...createsByTable.entries()].sort(([tableA], [tableB]) => {
+                    if (!schema)
+                        return 0;
+                    // Resolve schema keys from supabase names (strip prefix)
+                    const configA = getEngineConfig().tables.find((t) => t.supabaseName === tableA);
+                    const configB = getEngineConfig().tables.find((t) => t.supabaseName === tableB);
+                    const keyA = configA?.schemaKey || tableA;
+                    const keyB = configB?.schemaKey || tableB;
+                    const aIsChild = isChildTable(schema, keyA);
+                    const bIsChild = isChildTable(schema, keyB);
+                    if (aIsChild && !bIsChild)
+                        return 1;
+                    if (!aIsChild && bIsChild)
+                        return -1;
                     return 0;
-                // Resolve schema keys from supabase names (strip prefix)
-                const configA = getEngineConfig().tables.find((t) => t.supabaseName === tableA);
-                const configB = getEngineConfig().tables.find((t) => t.supabaseName === tableB);
-                const keyA = configA?.schemaKey || tableA;
-                const keyB = configB?.schemaKey || tableB;
-                const aIsChild = isChildTable(schema, keyA);
-                const bIsChild = isChildTable(schema, keyB);
-                if (aIsChild && !bIsChild)
-                    return 1;
-                if (!aIsChild && bIsChild)
-                    return -1;
-                return 0;
-            });
-            for (const [tableName, items] of sortedTableEntries) {
-                const supabase = getSupabase();
-                const deviceId = getDeviceId();
-                // Build batch payload — filter to schema-defined columns only
-                const payloads = items.map((item) => filterPayloadToSchema(tableName, {
-                    id: item.entityId,
-                    ...item.value,
-                    device_id: deviceId
-                }));
-                // Batch insert (up to 500 at a time to stay within Supabase limits)
-                const BATCH_SIZE = 500;
-                for (let i = 0; i < payloads.length; i += BATCH_SIZE) {
-                    const batch = payloads.slice(i, i + BATCH_SIZE);
-                    const batchItems = items.slice(i, i + BATCH_SIZE);
-                    try {
-                        debugLog(`[SYNC] Batch upsert ${batch.length} rows into ${tableName}`);
-                        const { error } = await supabase
-                            .from(tableName)
-                            .upsert(batch, { onConflict: 'id', ignoreDuplicates: false });
-                        if (error) {
-                            if (error.code === '23505') {
-                                // Duplicate key on a SECONDARY unique constraint (e.g., csv_import_hash,
-                                // teller_transaction_id). The primary `id` column is unique by UUID
-                                // generation, so this means another row with a different id already has
-                                // the same value for a secondary unique field.
-                                //
-                                // Strategy: query Supabase for which entity IDs already exist, remove those
-                                // from the queue (they're true duplicates), then retry with only the new ones.
-                                // This avoids the catastrophic individual fallback (500 sequential HTTP requests).
-                                debugLog(`[SYNC] Batch create hit secondary unique constraint for ${tableName} — filtering duplicates`);
-                                try {
-                                    // Query which IDs from this batch already exist in Supabase
-                                    const batchEntityIds = batchItems.map((item) => item.entityId);
-                                    const { data: existingRows } = await supabase
-                                        .from(tableName)
-                                        .select('id')
-                                        .in('id', batchEntityIds);
-                                    const existingIds = new Set((existingRows || []).map((r) => r.id));
-                                    // Remove already-synced items from queue
-                                    const duplicateQueueIds = batchItems
-                                        .filter((item) => existingIds.has(item.entityId) && item.id)
-                                        .map((item) => item.id);
-                                    if (duplicateQueueIds.length > 0) {
-                                        await bulkRemoveSyncItems(duplicateQueueIds);
-                                        processedAny = true;
-                                        actualPushed += duplicateQueueIds.length;
-                                        debugLog(`[SYNC] Removed ${duplicateQueueIds.length} already-synced items from queue`);
-                                    }
-                                    // Retry with only the truly new items
-                                    const newBatch = batch.filter((row) => !existingIds.has(row.id));
-                                    const newBatchItems = batchItems.filter((item) => !existingIds.has(item.entityId));
-                                    if (newBatch.length > 0) {
-                                        const { error: retryError } = await supabase
+                });
+                for (const [tableName, items] of sortedTableEntries) {
+                    const supabase = getSupabase();
+                    const deviceId = getDeviceId();
+                    // Build batch payload — filter to schema-defined columns only.
+                    // Ensure system field `deleted` is always present: create payloads
+                    // queued before the column defaulted locally (or by callers that
+                    // never set it) serialize with `undefined`/`null`, which violates
+                    // the Supabase NOT NULL constraint on `deleted`.
+                    const payloads = items.map((item) => {
+                        const rawPayload = {
+                            id: item.entityId,
+                            ...item.value,
+                            device_id: deviceId
+                        };
+                        ensureSystemFieldDefaults(rawPayload);
+                        return filterPayloadToSchema(tableName, rawPayload);
+                    });
+                    // Batch insert (up to 500 at a time to stay within Supabase limits)
+                    const BATCH_SIZE = 500;
+                    for (let i = 0; i < payloads.length; i += BATCH_SIZE) {
+                        const batch = payloads.slice(i, i + BATCH_SIZE);
+                        const batchItems = items.slice(i, i + BATCH_SIZE);
+                        try {
+                            debugLog(`[SYNC] Batch upsert ${batch.length} rows into ${tableName}`);
+                            const { error } = await supabase
+                                .from(tableName)
+                                .upsert(batch, { onConflict: 'id', ignoreDuplicates: false });
+                            if (error) {
+                                if (error.code === '23505') {
+                                    // Duplicate key on a SECONDARY unique constraint (e.g., csv_import_hash,
+                                    // teller_transaction_id). The primary `id` column is unique by UUID
+                                    // generation, so this means another row with a different id already has
+                                    // the same value for a secondary unique field.
+                                    //
+                                    // Strategy: query Supabase for which entity IDs already exist, remove those
+                                    // from the queue (they're true duplicates), then retry with only the new ones.
+                                    // This avoids the catastrophic individual fallback (500 sequential HTTP requests).
+                                    debugLog(`[SYNC] Batch create hit secondary unique constraint for ${tableName} — filtering duplicates`);
+                                    try {
+                                        // Query which IDs from this batch already exist in Supabase
+                                        const batchEntityIds = batchItems.map((item) => item.entityId);
+                                        const { data: existingRows } = await supabase
                                             .from(tableName)
-                                            .upsert(newBatch, { onConflict: 'id', ignoreDuplicates: false });
-                                        if (!retryError) {
-                                            const idsToRemove = newBatchItems
-                                                .filter((item) => item.id)
-                                                .map((item) => item.id);
-                                            if (idsToRemove.length > 0) {
-                                                await bulkRemoveSyncItems(idsToRemove);
-                                                processedAny = true;
-                                                actualPushed += idsToRemove.length;
+                                            .select('id')
+                                            .in('id', batchEntityIds);
+                                        const existingIds = new Set((existingRows || []).map((r) => r.id));
+                                        // Remove already-synced items from queue
+                                        const duplicateQueueIds = batchItems
+                                            .filter((item) => existingIds.has(item.entityId) && item.id)
+                                            .map((item) => item.id);
+                                        if (duplicateQueueIds.length > 0) {
+                                            await bulkRemoveSyncItems(duplicateQueueIds);
+                                            processedAny = true;
+                                            actualPushed += duplicateQueueIds.length;
+                                            debugLog(`[SYNC] Removed ${duplicateQueueIds.length} already-synced items from queue`);
+                                        }
+                                        // Retry with only the truly new items
+                                        const newBatch = batch.filter((row) => !existingIds.has(row.id));
+                                        const newBatchItems = batchItems.filter((item) => !existingIds.has(item.entityId));
+                                        if (newBatch.length > 0) {
+                                            const { error: retryError } = await supabase
+                                                .from(tableName)
+                                                .upsert(newBatch, { onConflict: 'id', ignoreDuplicates: false });
+                                            if (!retryError) {
+                                                const idsToRemove = newBatchItems
+                                                    .filter((item) => item.id)
+                                                    .map((item) => item.id);
+                                                if (idsToRemove.length > 0) {
+                                                    await bulkRemoveSyncItems(idsToRemove);
+                                                    processedAny = true;
+                                                    actualPushed += idsToRemove.length;
+                                                }
+                                                debugLog(`[SYNC] Batch create retry success: ${newBatch.length} new rows into ${tableName}`);
+                                            }
+                                            else {
+                                                // Retry still failed — likely another secondary constraint issue.
+                                                // Remove all items from queue to prevent infinite retry loops.
+                                                debugError(`[SYNC] Batch create retry failed for ${tableName} — removing from queue to prevent retry storm:`, retryError);
+                                                const allQueueIds = newBatchItems
+                                                    .filter((item) => item.id)
+                                                    .map((item) => item.id);
+                                                if (allQueueIds.length > 0) {
+                                                    await bulkRemoveSyncItems(allQueueIds);
+                                                    processedAny = true;
+                                                }
                                             }
-                                            debugLog(`[SYNC] Batch create retry success: ${newBatch.length} new rows into ${tableName}`);
                                         }
                                         else {
-                                            // Retry still failed — likely another secondary constraint issue.
-                                            // Remove all items from queue to prevent infinite retry loops.
-                                            debugError(`[SYNC] Batch create retry failed for ${tableName} — removing from queue to prevent retry storm:`, retryError);
-                                            const allQueueIds = newBatchItems
-                                                .filter((item) => item.id)
-                                                .map((item) => item.id);
-                                            if (allQueueIds.length > 0) {
-                                                await bulkRemoveSyncItems(allQueueIds);
+                                            debugLog(`[SYNC] All ${batch.length} items were duplicates — batch fully resolved`);
+                                        }
+                                    }
+                                    catch (filterError) {
+                                        // If the filter query itself fails, remove items to prevent retry storm
+                                        debugError(`[SYNC] Duplicate filter query failed for ${tableName} — removing from queue:`, filterError);
+                                        const allQueueIds = batchItems
+                                            .filter((item) => item.id)
+                                            .map((item) => item.id);
+                                        if (allQueueIds.length > 0) {
+                                            await bulkRemoveSyncItems(allQueueIds);
+                                            processedAny = true;
+                                        }
+                                    }
+                                }
+                                else {
+                                    // Non-duplicate error — fall back to individual to identify the problem row(s).
+                                    // Common cause: RLS on child tables when parent hasn't synced yet.
+                                    debugError(`[SYNC] Batch upsert failed for ${tableName}:`, error);
+                                    for (const item of batchItems) {
+                                        try {
+                                            await processSyncItem(item);
+                                            if (item.id) {
+                                                await removeSyncItem(item.id);
                                                 processedAny = true;
+                                                actualPushed++;
                                             }
                                         }
+                                        catch (itemError) {
+                                            handleSyncItemError(item, itemError);
+                                        }
                                     }
-                                    else {
-                                        debugLog(`[SYNC] All ${batch.length} items were duplicates — batch fully resolved`);
+                                }
+                            }
+                            else {
+                                // Batch succeeded — bulk-remove all items from queue in one transaction
+                                const idsToRemove = batchItems.filter((item) => item.id).map((item) => item.id);
+                                if (idsToRemove.length > 0) {
+                                    await bulkRemoveSyncItems(idsToRemove);
+                                    processedAny = true;
+                                    actualPushed += idsToRemove.length;
+                                }
+                                debugLog(`[SYNC] Batch upsert success: ${batch.length} rows into ${tableName}`);
+                            }
+                        }
+                        catch (batchError) {
+                            // Network-level failure — fall back to individual
+                            debugError(`[SYNC] Batch insert threw for ${tableName}:`, batchError);
+                            for (const item of batchItems) {
+                                try {
+                                    await processSyncItem(item);
+                                    if (item.id) {
+                                        await removeSyncItem(item.id);
+                                        processedAny = true;
+                                        actualPushed++;
                                     }
                                 }
-                                catch (filterError) {
-                                    // If the filter query itself fails, remove items to prevent retry storm
-                                    debugError(`[SYNC] Duplicate filter query failed for ${tableName} — removing from queue:`, filterError);
-                                    const allQueueIds = batchItems.filter((item) => item.id).map((item) => item.id);
-                                    if (allQueueIds.length > 0) {
-                                        await bulkRemoveSyncItems(allQueueIds);
+                                catch (itemError) {
+                                    handleSyncItemError(item, itemError);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            // ── Batch non-create operations: group by table and UPSERT in bulk ──
+            // For set/delete/increment on non-singleton tables, read the full local entity
+            // from IndexedDB and upsert in batches. This turns N sequential HTTP requests
+            // into ceil(N/500) batch calls. Singleton tables need special ID reconciliation
+            // and must be processed individually.
+            const batchableItems = nonCreateItems.filter((item) => !isSingletonTable(item.table));
+            const individualItems = nonCreateItems.filter((item) => isSingletonTable(item.table));
+            if (batchableItems.length > 0) {
+                // Bulk-read sync queue IDs for the still-queued check (same optimization as creates)
+                const batchQueueIds = batchableItems.filter((item) => item.id).map((item) => item.id);
+                const batchQueuedRows = await db.table('syncQueue').bulkGet(batchQueueIds);
+                const batchStillQueuedIds = new Set(batchQueuedRows
+                    .map((row, i) => (row ? batchQueueIds[i] : null))
+                    .filter((id) => id !== null));
+                // Group by table
+                const itemsByTable = new Map();
+                for (const item of batchableItems) {
+                    if (item.id && !batchStillQueuedIds.has(item.id))
+                        continue;
+                    const existing = itemsByTable.get(item.table) || [];
+                    existing.push(item);
+                    itemsByTable.set(item.table, existing);
+                }
+                for (const [tableName, items] of itemsByTable) {
+                    const supabase = getSupabase();
+                    const deviceId = getDeviceId();
+                    const dexieTable = getDexieTableName(tableName);
+                    // Build batch payload from local IndexedDB state (full entity rows).
+                    // Bulk-read all entities in one IndexedDB call instead of per-item reads.
+                    const entityIds = items.map((item) => item.entityId);
+                    const localEntities = await db.table(dexieTable).bulkGet(entityIds);
+                    const entityMap = new Map();
+                    localEntities.forEach((entity, i) => {
+                        if (entity)
+                            entityMap.set(entityIds[i], entity);
+                    });
+                    const payloads = [];
+                    const validItems = [];
+                    for (const item of items) {
+                        const localEntity = entityMap.get(item.entityId);
+                        if (!localEntity) {
+                            // Entity deleted locally — for delete ops this is expected (already gone),
+                            // for others skip it
+                            if (item.operationType === 'delete') {
+                                // Still need to ensure server-side deletion; fall back to individual
+                                try {
+                                    await processSyncItem(item);
+                                    if (item.id) {
+                                        await removeSyncItem(item.id);
                                         processedAny = true;
+                                        actualPushed++;
                                     }
                                 }
+                                catch (itemError) {
+                                    handleSyncItemError(item, itemError);
+                                }
                             }
-                            else {
-                                // Non-duplicate error — fall back to individual to identify the problem row(s).
-                                // Common cause: RLS on child tables when parent hasn't synced yet.
+                            continue;
+                        }
+                        // Strip internal Dexie fields, add device_id, filter to schema columns.
+                        // Ensure system field `deleted` is always present — IndexedDB rows created
+                        // before this field was set default to `undefined`, which serializes as `null`
+                        // and violates NOT NULL constraints on Supabase.
+                        const rawPayload = { ...localEntity, device_id: deviceId };
+                        ensureSystemFieldDefaults(rawPayload);
+                        delete rawPayload._version;
+                        payloads.push(filterPayloadToSchema(tableName, rawPayload));
+                        validItems.push(item);
+                    }
+                    if (payloads.length === 0)
+                        continue;
+                    const BATCH_SIZE = 500;
+                    for (let i = 0; i < payloads.length; i += BATCH_SIZE) {
+                        const batch = payloads.slice(i, i + BATCH_SIZE);
+                        const batchItems = validItems.slice(i, i + BATCH_SIZE);
+                        try {
+                            debugLog(`[SYNC] Batch upsert ${batch.length} rows into ${tableName}`);
+                            const { error } = await supabase
+                                .from(tableName)
+                                .upsert(batch, { onConflict: 'id', ignoreDuplicates: false });
+                            if (error) {
+                                // Batch failed — fall back to individual processing
                                 debugError(`[SYNC] Batch upsert failed for ${tableName}:`, error);
                                 for (const item of batchItems) {
                                     try {
@@ -1302,117 +1521,19 @@ async function pushPendingOps() {
                                     }
                                 }
                             }
-                        }
-                        else {
-                            // Batch succeeded — bulk-remove all items from queue in one transaction
-                            const idsToRemove = batchItems.filter((item) => item.id).map((item) => item.id);
-                            if (idsToRemove.length > 0) {
-                                await bulkRemoveSyncItems(idsToRemove);
-                                processedAny = true;
-                                actualPushed += idsToRemove.length;
-                            }
-                            debugLog(`[SYNC] Batch upsert success: ${batch.length} rows into ${tableName}`);
-                        }
-                    }
-                    catch (batchError) {
-                        // Network-level failure — fall back to individual
-                        debugError(`[SYNC] Batch insert threw for ${tableName}:`, batchError);
-                        for (const item of batchItems) {
-                            try {
-                                await processSyncItem(item);
-                                if (item.id) {
-                                    await removeSyncItem(item.id);
-                                    processedAny = true;
-                                    actualPushed++;
-                                }
-                            }
-                            catch (itemError) {
-                                handleSyncItemError(item, itemError);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        // ── Batch non-create operations: group by table and UPSERT in bulk ──
-        // For set/delete/increment on non-singleton tables, read the full local entity
-        // from IndexedDB and upsert in batches. This turns N sequential HTTP requests
-        // into ceil(N/500) batch calls. Singleton tables need special ID reconciliation
-        // and must be processed individually.
-        const batchableItems = nonCreateItems.filter((item) => !isSingletonTable(item.table));
-        const individualItems = nonCreateItems.filter((item) => isSingletonTable(item.table));
-        if (batchableItems.length > 0) {
-            // Bulk-read sync queue IDs for the still-queued check (same optimization as creates)
-            const batchQueueIds = batchableItems.filter((item) => item.id).map((item) => item.id);
-            const batchQueuedRows = await db.table('syncQueue').bulkGet(batchQueueIds);
-            const batchStillQueuedIds = new Set(batchQueuedRows
-                .map((row, i) => (row ? batchQueueIds[i] : null))
-                .filter((id) => id !== null));
-            // Group by table
-            const itemsByTable = new Map();
-            for (const item of batchableItems) {
-                if (item.id && !batchStillQueuedIds.has(item.id))
-                    continue;
-                const existing = itemsByTable.get(item.table) || [];
-                existing.push(item);
-                itemsByTable.set(item.table, existing);
-            }
-            for (const [tableName, items] of itemsByTable) {
-                const supabase = getSupabase();
-                const deviceId = getDeviceId();
-                const dexieTable = getDexieTableName(tableName);
-                // Build batch payload from local IndexedDB state (full entity rows).
-                // Bulk-read all entities in one IndexedDB call instead of per-item reads.
-                const entityIds = items.map((item) => item.entityId);
-                const localEntities = await db.table(dexieTable).bulkGet(entityIds);
-                const entityMap = new Map();
-                localEntities.forEach((entity, i) => {
-                    if (entity)
-                        entityMap.set(entityIds[i], entity);
-                });
-                const payloads = [];
-                const validItems = [];
-                for (const item of items) {
-                    const localEntity = entityMap.get(item.entityId);
-                    if (!localEntity) {
-                        // Entity deleted locally — for delete ops this is expected (already gone),
-                        // for others skip it
-                        if (item.operationType === 'delete') {
-                            // Still need to ensure server-side deletion; fall back to individual
-                            try {
-                                await processSyncItem(item);
-                                if (item.id) {
-                                    await removeSyncItem(item.id);
+                            else {
+                                // Batch succeeded — bulk-remove all items from queue in one transaction
+                                const idsToRemove = batchItems.filter((item) => item.id).map((item) => item.id);
+                                if (idsToRemove.length > 0) {
+                                    await bulkRemoveSyncItems(idsToRemove);
                                     processedAny = true;
-                                    actualPushed++;
+                                    actualPushed += idsToRemove.length;
                                 }
-                            }
-                            catch (itemError) {
-                                handleSyncItemError(item, itemError);
+                                debugLog(`[SYNC] Batch upsert success: ${batch.length} rows into ${tableName}`);
                             }
                         }
-                        continue;
-                    }
-                    // Strip internal Dexie fields, add device_id, filter to schema columns
-                    const rawPayload = { ...localEntity, device_id: deviceId };
-                    delete rawPayload._version;
-                    payloads.push(filterPayloadToSchema(tableName, rawPayload));
-                    validItems.push(item);
-                }
-                if (payloads.length === 0)
-                    continue;
-                const BATCH_SIZE = 500;
-                for (let i = 0; i < payloads.length; i += BATCH_SIZE) {
-                    const batch = payloads.slice(i, i + BATCH_SIZE);
-                    const batchItems = validItems.slice(i, i + BATCH_SIZE);
-                    try {
-                        debugLog(`[SYNC] Batch upsert ${batch.length} rows into ${tableName}`);
-                        const { error } = await supabase
-                            .from(tableName)
-                            .upsert(batch, { onConflict: 'id', ignoreDuplicates: false });
-                        if (error) {
-                            // Batch failed — fall back to individual processing
-                            debugError(`[SYNC] Batch upsert failed for ${tableName}:`, error);
+                        catch (batchError) {
+                            debugError(`[SYNC] Batch upsert threw for ${tableName}:`, batchError);
                             for (const item of batchItems) {
                                 try {
                                     await processSyncItem(item);
@@ -1427,62 +1548,57 @@ async function pushPendingOps() {
                                 }
                             }
                         }
-                        else {
-                            // Batch succeeded — bulk-remove all items from queue in one transaction
-                            const idsToRemove = batchItems.filter((item) => item.id).map((item) => item.id);
-                            if (idsToRemove.length > 0) {
-                                await bulkRemoveSyncItems(idsToRemove);
-                                processedAny = true;
-                                actualPushed += idsToRemove.length;
-                            }
-                            debugLog(`[SYNC] Batch upsert success: ${batch.length} rows into ${tableName}`);
-                        }
                     }
-                    catch (batchError) {
-                        debugError(`[SYNC] Batch upsert threw for ${tableName}:`, batchError);
-                        for (const item of batchItems) {
-                            try {
-                                await processSyncItem(item);
-                                if (item.id) {
-                                    await removeSyncItem(item.id);
-                                    processedAny = true;
-                                    actualPushed++;
-                                }
-                            }
-                            catch (itemError) {
-                                handleSyncItemError(item, itemError);
-                            }
+                }
+            }
+            // ── Process singleton table operations individually (need ID reconciliation) ──
+            for (const item of individualItems) {
+                try {
+                    if (item.id) {
+                        const stillQueued = await db.table('syncQueue').get(item.id);
+                        if (!stillQueued) {
+                            debugLog(`[SYNC] Skipping purged item: ${item.operationType} ${item.table}/${item.entityId}`);
+                            continue;
                         }
                     }
+                    debugLog(`[SYNC] Processing: ${item.operationType} ${item.table}/${item.entityId}`);
+                    await processSyncItem(item);
+                    if (item.id) {
+                        await removeSyncItem(item.id);
+                        processedAny = true;
+                        actualPushed++;
+                        debugLog(`[SYNC] Success: ${item.operationType} ${item.table}/${item.entityId}`);
+                    }
+                }
+                catch (error) {
+                    handleSyncItemError(item, error);
                 }
             }
+            // If we didn't process anything (all items in backoff), stop iterating
+            if (!processedAny)
+                break;
         }
-        // ── Process singleton table operations individually (need ID reconciliation) ──
-        for (const item of individualItems) {
+    }
+    finally {
+        if (progressMonitor) {
+            clearInterval(progressMonitor);
+            // Final count flush before clearing progress — ensures the UI lands on
+            // the exact remaining count rather than whatever the last tick showed.
+            // Wrapped in try/catch because this runs inside finally and must not
+            // mask an upstream error.
             try {
-                if (item.id) {
-                    const stillQueued = await db.table('syncQueue').get(item.id);
-                    if (!stillQueued) {
-                        debugLog(`[SYNC] Skipping purged item: ${item.operationType} ${item.table}/${item.entityId}`);
-                        continue;
-                    }
-                }
-                debugLog(`[SYNC] Processing: ${item.operationType} ${item.table}/${item.entityId}`);
-                await processSyncItem(item);
-                if (item.id) {
-                    await removeSyncItem(item.id);
-                    processedAny = true;
-                    actualPushed++;
-                    debugLog(`[SYNC] Success: ${item.operationType} ${item.table}/${item.entityId}`);
-                }
+                const finalLiveCount = await db.table('syncQueue').count();
+                syncStatusStore.setPendingCount(finalLiveCount);
+                const finalCompleted = Math.max(0, snapshotItems.length - finalLiveCount);
+                const delta = finalCompleted - lastReportedCompleted;
+                if (delta !== 0)
+                    syncStatusStore.advanceProgress(delta);
             }
-            catch (error) {
-                handleSyncItemError(item, error);
+            catch (err) {
+                debugWarn('[SYNC] Final progress flush failed:', err);
             }
+            syncStatusStore.clearProgress();
         }
-        // If we didn't process anything (all items in backoff), stop iterating
-        if (!processedAny)
-            break;
     }
     return { originalCount, coalescedCount, actualPushed };
 }
@@ -1615,11 +1731,11 @@ async function processSyncItem(item) {
             // INSERT the full entity payload with the originating device_id.
             // Uses .select('id').maybeSingle() to verify the row was actually created
             // (RLS can silently block inserts, returning success with no data).
-            const payload = filterPayloadToSchema(table, {
+            const payload = filterPayloadToSchema(table, ensureSystemFieldDefaults({
                 id: entityId,
                 ...value,
                 device_id: deviceId
-            });
+            }));
             const { data, error } = await supabase.from(table).insert(payload).select('id').maybeSingle();
             // Duplicate key = another device already created this entity.
             // For regular tables, this is a no-op (the entity exists, which is what we wanted).
@@ -1724,6 +1840,7 @@ async function processSyncItem(item) {
                     debugLog(`[SYNC] Increment fallback to insert for missing row: ${table}/${entityId}`);
                     const rawInsertPayload = { ...localInc, device_id: deviceId };
                     delete rawInsertPayload._version;
+                    ensureSystemFieldDefaults(rawInsertPayload);
                     const insertPayload = filterPayloadToSchema(table, rawInsertPayload);
                     const { error: insertError } = await supabase
                         .from(table)
@@ -1810,6 +1927,7 @@ async function processSyncItem(item) {
                     const rawSetPayload = { ...localEntity, device_id: deviceId };
                     // Remove Dexie internal keys
                     delete rawSetPayload._version;
+                    ensureSystemFieldDefaults(rawSetPayload);
                     const insertPayload = filterPayloadToSchema(table, rawSetPayload);
                     const { data: inserted, error: insertError } = await supabase
                         .from(table)
@@ -2040,7 +2158,9 @@ export async function runFullSync(quiet = false, skipPull = false) {
                 try {
                     // Don't pass postPushCursor - we want ALL changes since stored cursor
                     // The conflict resolution handles our own pushed changes via device_id check
-                    pullEgress = await withTimeout(pullRemoteChanges(), getSyncOperationTimeout(preflightItems.length), 'Pull remote changes');
+                    // Fixed 5-min timeout for pulls — pulls can be very large (initial hydration,
+                    // force sync, 20,000+ rows) and should not be capped by push-count heuristics.
+                    pullEgress = await withTimeout(pullRemoteChanges(), 300000, 'Pull remote changes');
                     pullSucceeded = true;
                 }
                 catch (pullError) {
@@ -2735,9 +2855,10 @@ export async function startSyncEngine() {
     // Initialize network status monitoring (idempotent)
     isOnline.init();
     // Subscribe to auth state changes.
-    // CRITICAL for iOS PWA: Safari aggressively kills background tabs, which can expire
-    // the Supabase session. When the user returns, TOKEN_REFRESHED fires and we need
-    // to restart realtime + trigger a sync to catch up on missed changes.
+    // SIGNED_IN: user just logged in — full sync + realtime restart needed.
+    // TOKEN_REFRESHED: routine refresh (~every 55 min) — only restart realtime if
+    // unhealthy. No sync needed: the Supabase JS client handles token rotation
+    // automatically, and the visibility handler catches missed changes on tab return.
     authStateUnsubscribe = supabase.auth.onAuthStateChange(async (event, session) => {
         debugLog(`[SYNC] Auth state change: ${event}`);
         if (event === 'SIGNED_OUT') {
@@ -2747,20 +2868,30 @@ export async function startSyncEngine() {
             syncStatusStore.setStatus('error');
             syncStatusStore.setError('Signed out', 'Please sign in to sync your data.');
         }
-        else if (event === 'SIGNED_IN' || event === 'TOKEN_REFRESHED') {
-            // User signed in or token refreshed - restart sync
-            debugLog('[SYNC] Auth restored - resuming sync');
+        else if (event === 'SIGNED_IN') {
+            // User just logged in — restart realtime + full sync to hydrate
+            debugLog('[SYNC] Signed in — resuming sync');
             if (navigator.onLine) {
-                // Clear any auth errors
                 syncStatusStore.reset();
-                // Restart realtime
                 if (session?.user?.id) {
                     startRealtimeSubscriptions(session.user.id);
+                    lastRealtimeStartAt = Date.now();
                 }
-                // Run a sync to push any pending changes
                 runFullSync(false).catch((e) => debugError('[SYNC] Auth-triggered sync failed:', e));
             }
         }
+        else if (event === 'TOKEN_REFRESHED') {
+            // Routine token refresh (~every 55 min). The Supabase JS client automatically
+            // uses the new token for subsequent requests — no sync needed.
+            // Only restart realtime if it's currently unhealthy (e.g. WebSocket dropped).
+            const realtimeHealthy = isRealtimeHealthy();
+            debugLog(`[SYNC] Token refreshed — realtime healthy: ${realtimeHealthy}, skipping sync`);
+            if (navigator.onLine && !realtimeHealthy && session?.user?.id) {
+                debugLog('[SYNC] Restarting unhealthy realtime after token refresh');
+                startRealtimeSubscriptions(session.user.id);
+                lastRealtimeStartAt = Date.now();
+            }
+        }
         // Delegate to app-level callback
         const config = getEngineConfig();
         if (config.onAuthStateChange) {
@@ -2890,6 +3021,7 @@ export async function startSyncEngine() {
         const userId = await getCurrentUserId();
         if (userId) {
             startRealtimeSubscriptions(userId);
+            lastRealtimeStartAt = Date.now();
         }
     };
     window.addEventListener('online', handleOnlineRef);
@@ -2907,26 +3039,52 @@ export async function startSyncEngine() {
         const wasHidden = !isTabVisible;
         isTabVisible = !document.hidden;
         syncStatusStore.setTabVisible(isTabVisible);
-        // Track when tab becomes hidden
+        // Tab becoming hidden → schedule realtime teardown after delay
         if (!isTabVisible) {
             tabHiddenAt = Date.now();
+            // Schedule realtime teardown to eliminate background heartbeat egress.
+            // The 15s delay avoids thrashing on quick tab switches (e.g. checking
+            // a notification and returning takes 8-12s).
+            if (!realtimeTeardownTimer) {
+                realtimeTeardownTimer = setTimeout(() => {
+                    realtimeTeardownTimer = null;
+                    realtimeWasTornDown = true;
+                    stopRealtimeSubscriptions();
+                    debugLog('[SYNC] Realtime torn down — tab hidden for 15s+');
+                }, REALTIME_TEARDOWN_DELAY_MS);
+                debugLog('[SYNC] Realtime teardown scheduled in 15s');
+            }
             return;
         }
-        // If tab just became visible, check if we should sync
-        if (wasHidden && isTabVisible && navigator.onLine) {
-            // Only sync if user was away for > configured minutes AND realtime is not healthy
-            // If realtime is connected, we're already up-to-date
+        // Tab becoming visible
+        if (wasHidden && isTabVisible) {
+            // Cancel teardown timer if it hasn't fired yet (quick tab switch)
+            if (realtimeTeardownTimer) {
+                clearTimeout(realtimeTeardownTimer);
+                realtimeTeardownTimer = null;
+                debugLog('[SYNC] Realtime teardown cancelled — tab visible before 15s');
+            }
+            // Restart realtime if it was torn down while hidden
+            if (realtimeWasTornDown && navigator.onLine) {
+                realtimeWasTornDown = false;
+                getCurrentUserId().then((userId) => {
+                    if (userId) {
+                        startRealtimeSubscriptions(userId);
+                        lastRealtimeStartAt = Date.now();
+                        debugLog('[SYNC] Restarting realtime — tab visible after teardown');
+                    }
+                });
+            }
+            if (!navigator.onLine)
+                return;
             const awayDuration = tabHiddenAt ? Date.now() - tabHiddenAt : 0;
             tabHiddenAt = null;
             if (awayDuration < getVisibilitySyncMinAwayMs()) {
                 debugLog(`[SYNC] Visibility sync skipped: away only ${Math.round(awayDuration / 1000)}s (min: ${Math.round(getVisibilitySyncMinAwayMs() / 1000)}s)`);
                 return;
             }
-            // Skip sync if realtime is healthy (we're already up-to-date)
-            if (isRealtimeHealthy()) {
-                debugLog('[SYNC] Visibility sync skipped: realtime is healthy');
-                return;
-            }
+            // Don't skip on isRealtimeHealthy() — we may have just torn down realtime,
+            // so it won't be healthy yet. The poll catches changes missed while hidden.
             // Clear any pending visibility sync
             if (visibilityDebounceTimeout) {
                 clearTimeout(visibilityDebounceTimeout);
@@ -2934,7 +3092,7 @@ export async function startSyncEngine() {
             // Debounce to prevent rapid syncs when user quickly switches tabs
             visibilityDebounceTimeout = setTimeout(() => {
                 visibilityDebounceTimeout = null;
-                runFullSync(true).catch((e) => debugError('[SYNC] Visibility sync failed:', e)); // Quiet - no error shown if it fails
+                runFullSync(true).catch((e) => debugError('[SYNC] Visibility sync failed:', e));
             }, VISIBILITY_SYNC_DEBOUNCE_MS);
         }
     };
@@ -2960,6 +3118,7 @@ export async function startSyncEngine() {
         });
         // Start realtime subscriptions
         startRealtimeSubscriptions(userId);
+        lastRealtimeStartAt = Date.now();
     }
     // Start the periodic background sync timer.
     // This is the polling fallback for when realtime subscriptions are down.
@@ -2971,7 +3130,15 @@ export async function startSyncEngine() {
         // This egress optimization is critical — without it, every open tab polls
         // the entire database every 15 minutes regardless of realtime status.
         if (navigator.onLine && isTabVisible && !isRealtimeHealthy()) {
-            runFullSync(true).catch((e) => debugError('[SYNC] Periodic sync failed:', e)); // Quiet background sync
+            // Grace period: if realtime was just (re)started, give it up to 60s to
+            // establish the WebSocket connection before falling back to polling.
+            const sinceRealtimeStart = Date.now() - lastRealtimeStartAt;
+            if (lastRealtimeStartAt > 0 && sinceRealtimeStart < 60000) {
+                debugLog(`[SYNC] Skipping periodic poll — realtime reconnecting (${Math.round(sinceRealtimeStart / 1000)}s ago)`);
+            }
+            else {
+                runFullSync(true).catch((e) => debugError('[SYNC] Periodic sync failed:', e));
+            }
         }
         // Cleanup old tombstones, conflict history, failed sync items, and recently modified cache
         await cleanupOldTombstones();
@@ -3116,6 +3283,10 @@ export async function stopSyncEngine() {
         clearTimeout(visibilityDebounceTimeout);
         visibilityDebounceTimeout = null;
     }
+    if (realtimeTeardownTimer) {
+        clearTimeout(realtimeTeardownTimer);
+        realtimeTeardownTimer = null;
+    }
     releaseSyncLock();
     _hasHydrated = false;
     _hydrationAttempted = false;