pushwork 1.0.11 → 1.0.15

package/src/core/sync-engine.ts

@@ -2,6 +2,7 @@ import {
   AutomergeUrl,
   Repo,
   DocHandle,
+  UrlHeads,
   parseAutomergeUrl,
   stringifyAutomergeUrl,
 } from "@automerge/automerge-repo";
@@ -33,6 +34,22 @@ import { ChangeDetector } from "./change-detection";
 import { MoveDetector } from "./move-detection";
 import { out } from "../utils/output";
 
+/**
+ * Apply a change to a document handle, using changeAt when heads are available
+ * to branch from a known version, otherwise falling back to change.
+ */
+function changeWithOptionalHeads<T>(
+  handle: DocHandle<T>,
+  heads: UrlHeads | undefined,
+  callback: A.ChangeFn<T>
+): void {
+  if (heads && heads.length > 0) {
+    handle.changeAt(heads, callback);
+  } else {
+    handle.change(callback);
+  }
+}
+
 /**
  * Sync configuration constants
  */
@@ -242,12 +259,17 @@ export class SyncEngine {
           }
 
           if (this.handlesByPath.size > 0) {
-            // Sort handles leaf-first (deepest paths first, then shallower)
-            const sortedHandles = this.sortHandlesLeafFirst();
-            await waitForSync(
-              sortedHandles,
-              this.config.sync_server_storage_id
-            );
+            // Sync level-by-level: deepest paths first, then shallower.
+            // Within each level, documents sync in parallel. But we wait
+            // for each level to complete before starting the next, ensuring
+            // children are on the server before their parent directories.
+            const levels = this.groupHandlesByDepthLevel();
+            for (const handlesAtLevel of levels) {
+              await waitForSync(
+                handlesAtLevel,
+                this.config.sync_server_storage_id
+              );
+            }
           }
 
           // Wait for bidirectional sync to stabilize.
@@ -587,33 +609,18 @@ export class SyncEngine {
       const heads = fromEntry.head;
 
       // Update both name and content (if content changed during move)
-      if (heads && heads.length > 0) {
-        handle.changeAt(heads, (doc: FileDocument) => {
-          doc.name = toFileName;
-
-          // If new content is provided, update it (handles move + modification case)
-          if (move.newContent !== undefined) {
-            if (typeof move.newContent === "string") {
-              doc.content = new A.ImmutableString(move.newContent);
-            } else {
-              doc.content = move.newContent;
-            }
+      changeWithOptionalHeads(handle, heads, (doc: FileDocument) => {
+        doc.name = toFileName;
+
+        // If new content is provided, update it (handles move + modification case)
+        if (move.newContent !== undefined) {
+          if (typeof move.newContent === "string") {
+            doc.content = new A.ImmutableString(move.newContent);
+          } else {
+            doc.content = move.newContent;
           }
-        });
-      } else {
-        handle.change((doc: FileDocument) => {
-          doc.name = toFileName;
-
-          // If new content is provided, update it (handles move + modification case)
-          if (move.newContent !== undefined) {
-            if (typeof move.newContent === "string") {
-              doc.content = new A.ImmutableString(move.newContent);
-            } else {
-              doc.content = move.newContent;
-            }
-          }
-        });
-      }
+        }
+      });
 
       // Get versioned URL after changes (includes current heads)
       const versionedUrl = this.getVersionedUrl(handle);
@@ -760,15 +767,9 @@ export class SyncEngine {
     if (snapshot && filePath) {
       heads = snapshot.files.get(filePath)?.head;
     }
-    if (heads) {
-      handle.changeAt(heads, (doc: FileDocument) => {
-        doc.content = new A.ImmutableString("");
-      });
-    } else {
-      handle.change((doc: FileDocument) => {
-        doc.content = new A.ImmutableString("");
-      });
-    }
+    changeWithOptionalHeads(handle, heads, (doc: FileDocument) => {
+      doc.content = new A.ImmutableString("");
+    });
   }
 
   /**
@@ -799,35 +800,19 @@ export class SyncEngine {
     let didChange = false;
     const snapshotEntry = snapshot.directories.get(directoryPath);
     const heads = snapshotEntry?.head;
-    if (heads) {
-      dirHandle.changeAt(heads, (doc: DirectoryDocument) => {
-        const existingIndex = doc.docs.findIndex(
-          (entry) => entry.name === fileName && entry.type === "file"
-        );
-        if (existingIndex === -1) {
-          doc.docs.push({
-            name: fileName,
-            type: "file",
-            url: fileUrl,
-          });
-          didChange = true;
-        }
-      });
-    } else {
-      dirHandle.change((doc: DirectoryDocument) => {
-        const existingIndex = doc.docs.findIndex(
-          (entry) => entry.name === fileName && entry.type === "file"
-        );
-        if (existingIndex === -1) {
-          doc.docs.push({
-            name: fileName,
-            type: "file",
-            url: fileUrl,
-          });
-          didChange = true;
-        }
-      });
-    }
+    changeWithOptionalHeads(dirHandle, heads, (doc: DirectoryDocument) => {
+      const existingIndex = doc.docs.findIndex(
+        (entry) => entry.name === fileName && entry.type === "file"
+      );
+      if (existingIndex === -1) {
+        doc.docs.push({
+          name: fileName,
+          type: "file",
+          url: fileUrl,
+        });
+        didChange = true;
+      }
+    });
     // Always track the directory (even if unchanged) for proper leaf-first sync ordering
     this.handlesByPath.set(directoryPath, dirHandle);
     
@@ -1007,37 +992,20 @@ export class SyncEngine {
       const heads = snapshotEntry?.head;
       let didChange = false;
 
-      if (heads) {
-        dirHandle.changeAt(heads, (doc: DirectoryDocument) => {
-          const indexToRemove = doc.docs.findIndex(
-            (entry) => entry.name === fileName && entry.type === "file"
-          );
-          if (indexToRemove !== -1) {
-            doc.docs.splice(indexToRemove, 1);
-            didChange = true;
-            out.taskLine(
-              `Removed ${fileName} from ${
-                formatRelativePath(directoryPath) || "root"
-              }`
-            );
-          }
-        });
-      } else {
-        dirHandle.change((doc: DirectoryDocument) => {
-          const indexToRemove = doc.docs.findIndex(
-            (entry) => entry.name === fileName && entry.type === "file"
+      changeWithOptionalHeads(dirHandle, heads, (doc: DirectoryDocument) => {
+        const indexToRemove = doc.docs.findIndex(
+          (entry) => entry.name === fileName && entry.type === "file"
+        );
+        if (indexToRemove !== -1) {
+          doc.docs.splice(indexToRemove, 1);
+          didChange = true;
+          out.taskLine(
+            `Removed ${fileName} from ${
+              formatRelativePath(directoryPath) || "root"
+            }`
           );
-          if (indexToRemove !== -1) {
-            doc.docs.splice(indexToRemove, 1);
-            didChange = true;
-            out.taskLine(
-              `Removed ${fileName} from ${
-                formatRelativePath(directoryPath) || "root"
-              }`
-            );
-          }
-        });
-      }
+        }
+      });
 
       if (didChange && snapshotEntry) {
         snapshotEntry.head = dirHandle.heads();
@@ -1184,15 +1152,9 @@ export class SyncEngine {
 
       const timestamp = Date.now();
 
-      if (heads) {
-        rootHandle.changeAt(heads, (doc: DirectoryDocument) => {
-          doc.lastSyncAt = timestamp;
-        });
-      } else {
-        rootHandle.change((doc: DirectoryDocument) => {
-          doc.lastSyncAt = timestamp;
-        });
-      }
+      changeWithOptionalHeads(rootHandle, heads, (doc: DirectoryDocument) => {
+        doc.lastSyncAt = timestamp;
+      });
 
       // Track root directory for network sync
       this.handlesByPath.set("", rootHandle);
@@ -1206,34 +1168,45 @@ export class SyncEngine {
   }
 
   /**
-   * Sort tracked handles leaf-first (deepest paths first).
-   * Returns handles in sorted order, logging URLs with heads for debugging.
+   * Group tracked handles by depth level, ordered leaf-first (deepest level first).
+   * Returns an array of arrays, where each inner array contains all handles at the
+   * same depth level. Levels are ordered from deepest to shallowest (root).
+   *
+   * This grouping enables level-by-level network sync: all documents at the deepest
+   * level sync in parallel first, then the next level up, etc. This ensures children
+   * are fully synced to the server before their parent directories.
    */
-  private sortHandlesLeafFirst(): DocHandle<unknown>[] {
-    // Sort paths by depth (descending - deepest first), then alphabetically
-    const sortedPaths = Array.from(this.handlesByPath.keys()).sort((a, b) => {
-      const depthA = a ? a.split("/").length : 0;
-      const depthB = b ? b.split("/").length : 0;
-
-      // Deepest first
-      if (depthA !== depthB) {
-        return depthB - depthA;
+  private groupHandlesByDepthLevel(): DocHandle<unknown>[][] {
+    // Group paths by depth
+    const pathsByDepth = new Map<number, string[]>();
+    for (const path of this.handlesByPath.keys()) {
+      const depth = path ? path.split("/").length : 0;
+      if (!pathsByDepth.has(depth)) {
+        pathsByDepth.set(depth, []);
       }
+      pathsByDepth.get(depth)!.push(path);
+    }
 
-      // Alphabetically by path
-      return a.localeCompare(b);
-    });
+    // Sort depths descending (deepest first) to get leaf-first ordering
+    const sortedDepths = Array.from(pathsByDepth.keys()).sort((a, b) => b - a);
 
-    // Log the sync order with versioned URLs for debugging (keep on complete)
-    const handles: DocHandle<unknown>[] = [];
-    for (const path of sortedPaths) {
-      const handle = this.handlesByPath.get(path)!;
-      const versionedUrl = this.getVersionedUrl(handle);
-      out.taskLine(`Sync: ${path || "(root)"} -> ${versionedUrl}`, true);
-      handles.push(handle);
+    // Build level groups, logging sync order for debugging
+    const levels: DocHandle<unknown>[][] = [];
+    for (const depth of sortedDepths) {
+      const paths = pathsByDepth.get(depth)!;
+      paths.sort((a, b) => a.localeCompare(b)); // Alphabetical within level
+
+      const handlesAtLevel: DocHandle<unknown>[] = [];
+      for (const path of paths) {
+        const handle = this.handlesByPath.get(path)!;
+        const versionedUrl = this.getVersionedUrl(handle);
+        out.taskLine(`Sync: ${path || "(root)"} -> ${versionedUrl}`, true);
+        handlesAtLevel.push(handle);
+      }
+      levels.push(handlesAtLevel);
     }
 
-    return handles;
+    return levels;
   }
 
   /**
@@ -1281,8 +1254,17 @@ export class SyncEngine {
       filesByDir.get(dirPath)!.push(filePath);
     }
 
-    // Process each directory that has files
-    for (const [dirPath, filePaths] of filesByDir.entries()) {
+    // Process directories leaf-first (deepest first) so children are
+    // up-to-date before their parents are processed
+    const sortedDirPaths = Array.from(filesByDir.keys()).sort((a, b) => {
+      const depthA = a ? a.split("/").length : 0;
+      const depthB = b ? b.split("/").length : 0;
+      if (depthA !== depthB) return depthB - depthA;
+      return a.localeCompare(b);
+    });
+
+    for (const dirPath of sortedDirPaths) {
+      const filePaths = filesByDir.get(dirPath)!;
       try {
         // Get the directory URL
         let dirUrl: AutomergeUrl;
@@ -1333,31 +1315,17 @@ export class SyncEngine {
 
         // Update all file entries in the directory document
         let didChange = false;
-        if (heads) {
-          dirHandle.changeAt(heads, (doc: DirectoryDocument) => {
-            for (const [fileName, newUrl] of fileUrlUpdates) {
-              const existingIndex = doc.docs.findIndex(
-                (entry) => entry.name === fileName && entry.type === "file"
-              );
-              if (existingIndex !== -1 && doc.docs[existingIndex].url !== newUrl) {
-                doc.docs[existingIndex].url = newUrl;
-                didChange = true;
-              }
-            }
-          });
-        } else {
-          dirHandle.change((doc: DirectoryDocument) => {
-            for (const [fileName, newUrl] of fileUrlUpdates) {
-              const existingIndex = doc.docs.findIndex(
-                (entry) => entry.name === fileName && entry.type === "file"
-              );
-              if (existingIndex !== -1 && doc.docs[existingIndex].url !== newUrl) {
-                doc.docs[existingIndex].url = newUrl;
-                didChange = true;
-              }
+        changeWithOptionalHeads(dirHandle, heads, (doc: DirectoryDocument) => {
+          for (const [fileName, newUrl] of fileUrlUpdates) {
+            const existingIndex = doc.docs.findIndex(
+              (entry) => entry.name === fileName && entry.type === "file"
+            );
+            if (existingIndex !== -1 && doc.docs[existingIndex].url !== newUrl) {
+              doc.docs[existingIndex].url = newUrl;
+              didChange = true;
             }
-          });
-        }
+          }
+        });
 
         // Track directory and update heads
         if (didChange) {
@@ -1452,29 +1420,15 @@ export class SyncEngine {
         const parentHeads = parentSnapshotEntry?.head;
 
         let didChange = false;
-        if (parentHeads) {
-          parentHandle.changeAt(parentHeads, (doc: DirectoryDocument) => {
-            const existingIndex = doc.docs.findIndex(
-              (entry) => entry.name === dirName && entry.type === "folder"
-            );
-            if (existingIndex !== -1) {
-              // Update the URL with current versioned URL
-              doc.docs[existingIndex].url = currentVersionedUrl;
-              didChange = true;
-            }
-          });
-        } else {
-          parentHandle.change((doc: DirectoryDocument) => {
-            const existingIndex = doc.docs.findIndex(
-              (entry) => entry.name === dirName && entry.type === "folder"
-            );
-            if (existingIndex !== -1) {
-              // Update the URL with current versioned URL
-              doc.docs[existingIndex].url = currentVersionedUrl;
-              didChange = true;
-            }
-          });
-        }
+        changeWithOptionalHeads(parentHandle, parentHeads, (doc: DirectoryDocument) => {
+          const existingIndex = doc.docs.findIndex(
+            (entry) => entry.name === dirName && entry.type === "folder"
+          );
+          if (existingIndex !== -1) {
+            doc.docs[existingIndex].url = currentVersionedUrl;
+            didChange = true;
+          }
+        });
 
         // Track parent for sync and update its heads in snapshot
         if (didChange) {

package/src/utils/network-sync.ts

@@ -1,4 +1,9 @@
-import { DocHandle, StorageId, Repo, AutomergeUrl } from "@automerge/automerge-repo";
+import {
+  DocHandle,
+  StorageId,
+  Repo,
+  AutomergeUrl,
+} from "@automerge/automerge-repo";
 import * as A from "@automerge/automerge";
 import { out } from "./output";
 import { DirectoryDocument } from "../types";
@@ -8,7 +13,7 @@ import { getPlainUrl } from "./directory";
  * Wait for bidirectional sync to stabilize.
  * This function waits until document heads stop changing, indicating that
  * both outgoing and incoming sync has completed.
- * 
+ *
  * @param repo - The Automerge repository
  * @param rootDirectoryUrl - The root directory URL to start traversal from
  * @param syncServerStorageId - The sync server storage ID
@@ -22,7 +27,7 @@ export async function waitForBidirectionalSync(
     timeoutMs?: number;
     pollIntervalMs?: number;
     stableChecksRequired?: number;
-  } = {}
+  } = {},
 ): Promise<void> {
   const {
     timeoutMs = 10000,
@@ -70,7 +75,7 @@ export async function waitForBidirectionalSync(
  */
 async function getAllDocumentHeads(
   repo: Repo,
-  rootDirectoryUrl: AutomergeUrl
+  rootDirectoryUrl: AutomergeUrl,
 ): Promise<Map<string, string>> {
   const heads = new Map<string, string>();
   // Pass URL as-is; collectHeadsRecursive will strip heads
@@ -85,13 +90,13 @@ async function getAllDocumentHeads(
 async function collectHeadsRecursive(
   repo: Repo,
   directoryUrl: AutomergeUrl,
-  heads: Map<string, string>
+  heads: Map<string, string>,
 ): Promise<void> {
   try {
     const plainUrl = getPlainUrl(directoryUrl);
     const handle = await repo.find<DirectoryDocument>(plainUrl);
     const doc = await handle.doc();
-    
+
     // Record this directory's heads (use plain URL as key for consistency)
     heads.set(plainUrl, JSON.stringify(handle.heads()));
 
@@ -125,7 +130,7 @@ async function collectHeadsRecursive(
  */
 function headsMapEqual(
   a: Map<string, string>,
-  b: Map<string, string>
+  b: Map<string, string>,
 ): boolean {
   if (a.size !== b.size) {
     return false;
@@ -144,7 +149,7 @@ function headsMapEqual(
 export async function waitForSync(
   handlesToWaitOn: DocHandle<unknown>[],
   syncServerStorageId?: StorageId,
-  timeoutMs: number = 60000
+  timeoutMs: number = 1000000,
 ): Promise<void> {
   const startTime = Date.now();
 
@@ -192,8 +197,8 @@ export async function waitForSync(
         cleanup();
         reject(
           new Error(
-            `Sync timeout after ${timeoutMs}ms for document ${handle.url}`
-          )
+            `Sync timeout after ${timeoutMs}ms for document ${handle.url}`,
+          ),
         );
       }, timeoutMs);