pushwork 1.0.5 → 1.0.11

package/src/utils/network-sync.ts

@@ -1,169 +1,247 @@
-import { DocHandle, StorageId } from "@automerge/automerge-repo";
+import { DocHandle, StorageId, Repo, AutomergeUrl } from "@automerge/automerge-repo";
 import * as A from "@automerge/automerge";
+import { out } from "./output";
+import { DirectoryDocument } from "../types";
+import { getPlainUrl } from "./directory";
+
+/**
+ * Wait for bidirectional sync to stabilize.
+ * This function waits until document heads stop changing, indicating that
+ * both outgoing and incoming sync has completed.
+ * 
+ * @param repo - The Automerge repository
+ * @param rootDirectoryUrl - The root directory URL to start traversal from
+ * @param syncServerStorageId - The sync server storage ID
+ * @param options - Configuration options
+ */
+export async function waitForBidirectionalSync(
+  repo: Repo,
+  rootDirectoryUrl: AutomergeUrl | undefined,
+  syncServerStorageId: StorageId | undefined,
+  options: {
+    timeoutMs?: number;
+    pollIntervalMs?: number;
+    stableChecksRequired?: number;
+  } = {}
+): Promise<void> {
+  const {
+    timeoutMs = 10000,
+    pollIntervalMs = 100,
+    stableChecksRequired = 3,
+  } = options;
+
+  if (!syncServerStorageId || !rootDirectoryUrl) {
+    return;
+  }
+
+  const startTime = Date.now();
+  let lastSeenHeads = new Map<string, string>();
+  let stableCount = 0;
+
+  while (Date.now() - startTime < timeoutMs) {
+    // Get current heads for all documents in the directory hierarchy
+    const currentHeads = await getAllDocumentHeads(repo, rootDirectoryUrl);
+
+    // Check if heads are stable (no changes since last check)
+    const isStable = headsMapEqual(lastSeenHeads, currentHeads);
+
+    if (isStable) {
+      stableCount++;
+      if (stableCount >= stableChecksRequired) {
+        return; // Converged!
+      }
+    } else {
+      stableCount = 0;
+      lastSeenHeads = currentHeads;
+    }
+
+    await new Promise((r) => setTimeout(r, pollIntervalMs));
+  }
+
+  // Timeout - but don't throw, just log a warning
+  // The sync may still work, we just couldn't confirm stability
+  out.taskLine(`Sync stability check timed out after ${timeoutMs}ms`, true);
+}
+
+/**
+ * Get all document heads in the directory hierarchy.
+ * Returns a map of document URL -> serialized heads.
+ * Uses plain URLs (without heads) to ensure we see current document state.
+ */
+async function getAllDocumentHeads(
+  repo: Repo,
+  rootDirectoryUrl: AutomergeUrl
+): Promise<Map<string, string>> {
+  const heads = new Map<string, string>();
+  // Pass URL as-is; collectHeadsRecursive will strip heads
+  await collectHeadsRecursive(repo, rootDirectoryUrl, heads);
+  return heads;
+}
+
+/**
+ * Recursively collect document heads from the directory hierarchy.
+ * Uses getPlainUrl to strip heads and always see the CURRENT state of documents.
+ */
+async function collectHeadsRecursive(
+  repo: Repo,
+  directoryUrl: AutomergeUrl,
+  heads: Map<string, string>
+): Promise<void> {
+  try {
+    const plainUrl = getPlainUrl(directoryUrl);
+    const handle = await repo.find<DirectoryDocument>(plainUrl);
+    const doc = await handle.doc();
+    
+    // Record this directory's heads (use plain URL as key for consistency)
+    heads.set(plainUrl, JSON.stringify(handle.heads()));
+
+    if (!doc || !doc.docs) {
+      return;
+    }
+
+    // Process all entries in the directory
+    for (const entry of doc.docs) {
+      if (entry.type === "folder") {
+        // Recurse into subdirectory (entry.url may have stale heads)
+        await collectHeadsRecursive(repo, entry.url, heads);
+      } else if (entry.type === "file") {
+        // Get file document heads (strip heads from entry.url)
+        try {
+          const fileUrl = getPlainUrl(entry.url);
+          const fileHandle = await repo.find(fileUrl);
+          heads.set(fileUrl, JSON.stringify(fileHandle.heads()));
+        } catch {
+          // File document may not exist yet
+        }
+      }
+    }
+  } catch {
+    // Directory may not exist yet
+  }
+}
+
+/**
+ * Compare two heads maps for equality.
+ */
+function headsMapEqual(
+  a: Map<string, string>,
+  b: Map<string, string>
+): boolean {
+  if (a.size !== b.size) {
+    return false;
+  }
+  for (const [key, value] of a) {
+    if (b.get(key) !== value) {
+      return false;
+    }
+  }
+  return true;
+}
 
 /**
  * Wait for documents to sync to the remote server
- * Based on patchwork-cli implementation with timeout for debugging
  */
 export async function waitForSync(
   handlesToWaitOn: DocHandle<unknown>[],
   syncServerStorageId?: StorageId,
-  timeoutMs: number = 60000 // 60 second timeout for debugging
+  timeoutMs: number = 60000
 ): Promise<void> {
   const startTime = Date.now();
 
   if (!syncServerStorageId) {
-    console.warn(
-      "No sync server storage ID provided. Skipping network sync wait."
-    );
+    // No sync server storage ID - skip network sync
     return;
   }
 
   if (handlesToWaitOn.length === 0) {
-    console.log("🔄 No documents to sync");
+    // No documents to sync
     return;
   }
 
-  // Debug logging only in verbose mode (can be controlled via env var later)
-  const verbose = false;
-
-  if (verbose) {
-    console.log(
-      `🔄 Waiting for ${handlesToWaitOn.length} documents to sync...`
-    );
-    console.log(`📡 Using sync server storage ID: ${syncServerStorageId}`);
-
-    handlesToWaitOn.forEach((handle, i) => {
-      const localHeads = handle.heads();
-      const syncInfo = handle.getSyncInfo(syncServerStorageId);
-      const remoteHeads = syncInfo?.lastHeads;
-      console.log(`  📄 Document ${i + 1}: ${handle.url}`);
-      console.log(`    🏠 Local heads: ${JSON.stringify(localHeads)}`);
-      console.log(`    🌐 Remote heads: ${JSON.stringify(remoteHeads)}`);
-      console.log(
-        `    ✅ Already synced: ${A.equals(localHeads, remoteHeads)}`
-      );
-    });
-  }
+  let alreadySynced = 0;
+
+  const promises = handlesToWaitOn.map((handle) => {
+    // Check if already synced
+    const heads = handle.heads();
+    const syncInfo = handle.getSyncInfo(syncServerStorageId);
+    const remoteHeads = syncInfo?.lastHeads;
+    const wasAlreadySynced = A.equals(heads, remoteHeads);
 
-  const promises = handlesToWaitOn.map(
-    (handle, index) =>
-      new Promise<void>((resolve, reject) => {
-        let pollInterval: NodeJS.Timeout;
-
-        const timeout = setTimeout(() => {
-          clearInterval(pollInterval);
-          const localHeads = handle.heads();
-          const syncInfo = handle.getSyncInfo(syncServerStorageId);
-          const remoteHeads = syncInfo?.lastHeads;
-          console.log(`⏰ TIMEOUT for document ${index + 1}: ${handle.url}`);
-          console.log(`  Final local heads: ${JSON.stringify(localHeads)}`);
-          console.log(`  Final remote heads: ${JSON.stringify(remoteHeads)}`);
-          reject(
-            new Error(
-              `Sync timeout after ${timeoutMs}ms for document ${handle.url}`
-            )
-          );
-        }, timeoutMs);
-
-        const checkSync = () => {
-          const newHeads = handle.heads();
-          const syncInfo = handle.getSyncInfo(syncServerStorageId);
-          const remoteHeads = syncInfo?.lastHeads;
-
-          if (verbose) {
-            console.log(`🔍 Checking sync for ${handle.url}:`);
-            console.log(`  Local heads: ${JSON.stringify(newHeads)}`);
-            console.log(`  Remote heads: ${JSON.stringify(remoteHeads)}`);
-            console.log(`  Heads equal: ${A.equals(newHeads, remoteHeads)}`);
-          }
-
-          // If the remote heads are already up to date, we can resolve immediately
-          if (A.equals(newHeads, remoteHeads)) {
-            if (verbose) {
-              console.log(`✅ Document ${index + 1} synced: ${handle.url}`);
-            }
-            clearTimeout(timeout);
-            clearInterval(pollInterval);
-            resolve();
-            return true;
-          }
-          return false;
-        };
-
-        // Check if already synced
-        if (checkSync()) {
-          return;
+    if (wasAlreadySynced) {
+      alreadySynced++;
+      return Promise.resolve();
+    }
+
+    // Wait for convergence
+    return new Promise<void>((resolve, reject) => {
+      // TODO: can we delete this polling?
+      let pollInterval: NodeJS.Timeout;
+
+      const cleanup = () => {
+        clearTimeout(timeout);
+        clearInterval(pollInterval);
+        handle.off("remote-heads", onRemoteHeads);
+      };
+
+      const onConverged = () => {
+        cleanup();
+        resolve();
+      };
+
+      const timeout = setTimeout(() => {
+        cleanup();
+        reject(
+          new Error(
+            `Sync timeout after ${timeoutMs}ms for document ${handle.url}`
+          )
+        );
+      }, timeoutMs);
+
+      const isConverged = () => {
+        const localHeads = handle.heads();
+        const info = handle.getSyncInfo(syncServerStorageId);
+        return A.equals(localHeads, info?.lastHeads);
+      };
+
+      const onRemoteHeads = ({
+        storageId,
+      }: {
+        storageId: StorageId;
+        heads: any;
+      }) => {
+        if (storageId === syncServerStorageId && isConverged()) {
+          onConverged();
         }
+      };
 
-        // Periodically re-check if heads have converged (polling fallback)
-        pollInterval = setInterval(() => {
-          if (checkSync()) {
-            clearInterval(pollInterval);
-          }
-        }, 100); // Check every 100ms for faster response
-
-        // Also wait for remote-heads event (faster when events work)
-        const onRemoteHeads = ({
-          storageId,
-          heads,
-        }: {
-          storageId: StorageId;
-          heads: any;
-        }) => {
-          if (verbose) {
-            console.log(`📡 Received remote heads event for ${handle.url}:`);
-            console.log(`  Event storage ID: ${storageId}`);
-            console.log(`  Expected storage ID: ${syncServerStorageId}`);
-            console.log(`  Event heads: ${JSON.stringify(heads)}`);
-            console.log(
-              `  Current local heads: ${JSON.stringify(handle.heads())}`
-            );
-          }
-
-          if (
-            storageId === syncServerStorageId &&
-            A.equals(handle.heads(), heads)
-          ) {
-            if (verbose) {
-              console.log(
-                `✅ Document ${index + 1} synced via event: ${handle.url}`
-              );
-            }
-            clearTimeout(timeout);
-            clearInterval(pollInterval);
-            handle.off("remote-heads", onRemoteHeads);
-            resolve();
-          } else if (verbose) {
-            console.log(`❌ Heads/storage mismatch for ${handle.url}`);
-          }
-        };
-
-        if (verbose) {
-          console.log(`👂 Listening for remote-heads events on ${handle.url}`);
+      const poll = () => {
+        if (isConverged()) {
+          onConverged();
+          return true;
         }
-        handle.on("remote-heads", onRemoteHeads);
-      })
-  );
+        return false;
+      };
+
+      // Initial check
+      if (poll()) {
+        return;
+      }
+
+      // Start polling and event listening
+      pollInterval = setInterval(() => {
+        poll();
+      }, 100);
+
+      handle.on("remote-heads", onRemoteHeads);
+    });
+  });
 
   try {
     await Promise.all(promises);
-    const elapsed = Date.now() - startTime;
-    if (verbose) {
-      console.log(`✅ All documents synced to network (took ${elapsed}ms)`);
-    }
   } catch (error) {
     const elapsed = Date.now() - startTime;
-    console.error(`❌ Sync wait failed after ${elapsed}ms: ${error}`);
+    out.errorBlock("FAILED", `after ${elapsed}ms`);
+    out.crash(error);
     throw error;
   }
 }
-
-/**
- * Get the storage ID for the sync server
- * Using the same ID as patchwork-cli for consistency
- */
-export function getSyncServerStorageId(customStorageId?: string): StorageId {
-  return (customStorageId ||
-    "3760df37-a4c6-4f66-9ecd-732039a9385d") as StorageId;
-}