pushwork 1.0.20 → 1.0.22

package/src/core/sync-engine.ts

@@ -248,6 +248,111 @@ export class SyncEngine {
 		}
 	}
 
+	/**
+	 * Push local changes to server without pulling remote changes.
+	 * Detect changes, push to Automerge docs, upload to server. No bidirectional wait, no pull.
+	 */
+	async pushToRemote(): Promise<SyncResult> {
+		const result: SyncResult = {
+			success: false,
+			filesChanged: 0,
+			directoriesChanged: 0,
+			errors: [],
+			warnings: [],
+		}
+
+		// Reset tracked handles for sync
+		this.handlesByPath = new Map()
+
+		try {
+			const snapshot =
+				(await this.snapshotManager.load()) ||
+				this.snapshotManager.createEmpty()
+
+			debug(`pushToRemote: rootDirectoryUrl=${snapshot.rootDirectoryUrl?.slice(0, 30)}..., files=${snapshot.files.size}, dirs=${snapshot.directories.size}`)
+
+			// Detect all changes
+			debug("pushToRemote: detecting changes")
+			out.update("Detecting local changes")
+			const changes = await this.changeDetector.detectChanges(snapshot)
+
+			// Detect moves
+			const {moves, remainingChanges} = await this.moveDetector.detectMoves(
+				changes,
+				snapshot
+			)
+
+			debug(`pushToRemote: detected ${changes.length} changes, ${moves.length} moves, ${remainingChanges.length} remaining`)
+
+			// Push local changes to remote
+			debug("pushToRemote: pushing local changes")
+			const pushResult = await this.pushLocalChanges(
+				remainingChanges,
+				moves,
+				snapshot
+			)
+
+			result.filesChanged += pushResult.filesChanged
+			result.directoriesChanged += pushResult.directoriesChanged
+			result.errors.push(...pushResult.errors)
+			result.warnings.push(...pushResult.warnings)
+
+			debug(`pushToRemote: push complete - ${pushResult.filesChanged} files, ${pushResult.directoriesChanged} dirs changed`)
+
+			// Touch root directory if any changes were made
+			const hasChanges =
+				result.filesChanged > 0 || result.directoriesChanged > 0
+			if (hasChanges) {
+				await this.touchRootDirectory(snapshot)
+			}
+
+			// Wait for network sync (upload to server)
+			if (this.config.sync_enabled) {
+				try {
+					// Ensure root directory handle is tracked for sync
+					if (snapshot.rootDirectoryUrl) {
+						const rootHandle =
+							await this.repo.find<DirectoryDocument>(
+								snapshot.rootDirectoryUrl
+							)
+						this.handlesByPath.set("", rootHandle)
+					}
+
+					if (this.handlesByPath.size > 0) {
+						const allHandles = Array.from(
+							this.handlesByPath.values()
+						)
+						debug(`pushToRemote: waiting for ${allHandles.length} handles to sync to server`)
+						out.update(`Uploading ${allHandles.length} documents to sync server`)
+						await waitForSync(
+							allHandles,
+							this.config.sync_server_storage_id
+						)
+						debug("pushToRemote: all handles synced to server")
+					}
+				} catch (error) {
+					debug(`pushToRemote: network sync error: ${error}`)
+					out.taskLine(`Network sync failed: ${error}`, true)
+					result.warnings.push(`Network sync failed: ${error}`)
+				}
+			}
+
+			// Save updated snapshot
+			await this.snapshotManager.save(snapshot)
+
+			result.success = result.errors.length === 0
+			return result
+		} catch (error) {
+			result.errors.push({
+				path: "push",
+				operation: "push-to-remote",
+				error: error as Error,
+				recoverable: false,
+			})
+			return result
+		}
+	}
+
 	/**
 	 * Run full bidirectional sync
 	 */
@@ -275,6 +380,7 @@ export class SyncEngine {
 			// Wait for initial sync to receive any pending remote changes
 			if (this.config.sync_enabled && snapshot.rootDirectoryUrl) {
 				debug("sync: waiting for initial bidirectional sync")
+				out.update("Waiting for initial sync from server")
 				try {
 					await waitForBidirectionalSync(
 						this.repo,
@@ -293,6 +399,7 @@ export class SyncEngine {
 
 			// Detect all changes
 			debug("sync: detecting changes")
+			out.update("Detecting local and remote changes")
 			const changes = await this.changeDetector.detectChanges(snapshot)
 
 			// Detect moves
@@ -335,15 +442,21 @@ export class SyncEngine {
 						const allHandles = Array.from(
 							this.handlesByPath.values()
 						)
-						debug(`sync: waiting for ${allHandles.length} handles to sync to server`)
+						const handlePaths = Array.from(this.handlesByPath.keys())
+						debug(`sync: waiting for ${allHandles.length} handles to sync to server: ${handlePaths.slice(0, 10).map(p => p || "(root)").join(", ")}${handlePaths.length > 10 ? ` ...and ${handlePaths.length - 10} more` : ""}`)
+						out.update(`Uploading ${allHandles.length} documents to sync server`)
 						await waitForSync(
 							allHandles,
 							this.config.sync_server_storage_id
 						)
+						debug("sync: all handles synced to server")
 					}
 
 					// Wait for bidirectional sync to stabilize
-					debug("sync: waiting for bidirectional sync to stabilize")
+					// Use tracked handles for post-push check (cheaper than full tree scan)
+					const changedHandles = Array.from(this.handlesByPath.values())
+					debug(`sync: waiting for bidirectional sync to stabilize (${changedHandles.length} tracked handles)`)
+					out.update("Waiting for bidirectional sync to stabilize")
 					await waitForBidirectionalSync(
 						this.repo,
 						snapshot.rootDirectoryUrl,
@@ -352,6 +465,7 @@ export class SyncEngine {
 							timeoutMs: BIDIRECTIONAL_SYNC_TIMEOUT_MS,
 							pollIntervalMs: 100,
 							stableChecksRequired: 3,
+							handles: changedHandles.length > 0 ? changedHandles : undefined,
 						}
 					)
 				} catch (error) {
@@ -371,6 +485,9 @@ export class SyncEngine {
 			)
 
 			debug(`sync: phase 2 - pulling ${freshRemoteChanges.length} remote changes`)
+			if (freshRemoteChanges.length > 0) {
+				out.update(`Pulling ${freshRemoteChanges.length} remote changes`)
+			}
 			// Phase 2: Pull remote changes to local using fresh detection
 			const phase2Result = await this.pullRemoteChanges(
 				freshRemoteChanges,
@@ -460,11 +577,19 @@ export class SyncEngine {
 		}
 
 		// Process moves first - all detected moves are applied
-		for (const move of moves) {
+		if (moves.length > 0) {
+			debug(`push: processing ${moves.length} moves`)
+			out.update(`Processing ${moves.length} move${moves.length > 1 ? "s" : ""}`)
+		}
+		for (let i = 0; i < moves.length; i++) {
+			const move = moves[i]
 			try {
+				debug(`push: move ${i + 1}/${moves.length}: ${move.fromPath} -> ${move.toPath}`)
+				out.taskLine(`Moving ${move.fromPath} -> ${move.toPath}`)
 				await this.applyMoveToRemote(move, snapshot)
 				result.filesChanged++
 			} catch (error) {
+				debug(`push: move failed for ${move.fromPath}: ${error}`)
 				result.errors.push({
 					path: move.fromPath,
 					operation: "move",
@@ -481,7 +606,16 @@ export class SyncEngine {
 				c.changeType === ChangeType.BOTH_CHANGED
 		)
 
-		if (localChanges.length === 0) return result
+		if (localChanges.length === 0) {
+			debug("push: no local changes to push")
+			return result
+		}
+
+		const newFiles = localChanges.filter(c => !snapshot.files.has(c.path) && c.localContent !== null)
+		const modifiedFiles = localChanges.filter(c => snapshot.files.has(c.path) && c.localContent !== null)
+		const deletedFiles = localChanges.filter(c => c.localContent === null && snapshot.files.has(c.path))
+		debug(`push: ${localChanges.length} local changes (${newFiles.length} new, ${modifiedFiles.length} modified, ${deletedFiles.length} deleted)`)
+		out.update(`Pushing ${localChanges.length} local changes (${newFiles.length} new, ${modifiedFiles.length} modified, ${deletedFiles.length} deleted)`)
 
 		// Group changes by parent directory path
 		const changesByDir = new Map<string, DetectedChange[]>()
@@ -517,11 +651,20 @@ export class SyncEngine {
 			return depthB - depthA
 		})
 
+		debug(`push: processing ${sortedDirPaths.length} directories (deepest first)`)
+
 		// Track which directories were modified (for subdirectory URL propagation)
 		const modifiedDirs = new Set<string>()
+		let filesProcessed = 0
+		const totalFiles = localChanges.length
 
 		for (const dirPath of sortedDirPaths) {
 			const dirChanges = changesByDir.get(dirPath) || []
+			const dirLabel = dirPath || "(root)"
+
+			if (dirChanges.length > 0) {
+				debug(`push: directory "${dirLabel}": ${dirChanges.length} file changes`)
+			}
 
 			// Ensure directory document exists
 			if (snapshot.rootDirectoryUrl) {
@@ -536,10 +679,13 @@ export class SyncEngine {
 			for (const change of dirChanges) {
 				const fileName = change.path.split("/").pop() || ""
 				const snapshotEntry = snapshot.files.get(change.path)
+				filesProcessed++
 
 				try {
 					if (change.localContent === null && snapshotEntry) {
 						// Delete file
+						debug(`push: [${filesProcessed}/${totalFiles}] delete ${change.path}`)
+						out.update(`Pushing local changes [${filesProcessed}/${totalFiles}] deleting ${change.path}`)
 						await this.deleteRemoteFile(
 							snapshotEntry.url,
 							snapshot,
@@ -550,6 +696,8 @@ export class SyncEngine {
 						result.filesChanged++
 					} else if (!snapshotEntry) {
 						// New file
+						debug(`push: [${filesProcessed}/${totalFiles}] create ${change.path} (${change.fileType})`)
+						out.update(`Pushing local changes [${filesProcessed}/${totalFiles}] creating ${change.path}`)
 						const handle = await this.createRemoteFile(change)
 						if (handle) {
 							const entryUrl = this.getEntryUrl(handle, change.path)
@@ -569,9 +717,15 @@ export class SyncEngine {
 								}
 							)
 							result.filesChanged++
+							debug(`push: created ${change.path} -> ${handle.url.slice(0, 20)}...`)
 						}
 					} else {
 						// Update existing file
+						const contentSize = typeof change.localContent === "string"
+							? `${change.localContent!.length} chars`
+							: `${(change.localContent as Uint8Array).length} bytes`
+						debug(`push: [${filesProcessed}/${totalFiles}] update ${change.path} (${contentSize})`)
+						out.update(`Pushing local changes [${filesProcessed}/${totalFiles}] updating ${change.path}`)
 						await this.updateRemoteFile(
 							snapshotEntry.url,
 							change.localContent!,
@@ -593,6 +747,8 @@ export class SyncEngine {
 						result.filesChanged++
 					}
 				} catch (error) {
+					debug(`push: error processing ${change.path}: ${error}`)
+					out.taskLine(`Error pushing ${change.path}: ${error}`, true)
 					result.errors.push({
 						path: change.path,
 						operation: "local-to-remote",
@@ -631,6 +787,7 @@ export class SyncEngine {
 				deletedNames.length > 0 ||
 				subdirUpdates.length > 0
 			if (hasChanges && snapshot.rootDirectoryUrl) {
+				debug(`push: batch-updating directory "${dirLabel}" (+${newEntries.length} new, ~${updatedEntries.length} updated, -${deletedNames.length} deleted, ${subdirUpdates.length} subdir URL updates)`)
 				await this.batchUpdateDirectory(
 					snapshot,
 					dirPath,
@@ -644,6 +801,7 @@ export class SyncEngine {
 			}
 		}
 
+		debug(`push: complete - ${result.filesChanged} files, ${result.directoriesChanged} dirs changed, ${result.errors.length} errors`)
 		return result
 	}
 

package/src/types/config.ts

@@ -101,6 +101,11 @@ export interface StatusOptions extends CommandOptions {
   verbose?: boolean;
 }
 
+/**
+ * Push command specific options
+ */
+export interface PushOptions extends CommandOptions {}
+
 /**
  * Watch command specific options
  */

package/src/utils/network-sync.ts

@@ -32,12 +32,14 @@ export async function waitForBidirectionalSync(
     timeoutMs?: number;
     pollIntervalMs?: number;
     stableChecksRequired?: number;
+    handles?: DocHandle<unknown>[];
   } = {},
 ): Promise<void> {
   const {
     timeoutMs = 10000,
     pollIntervalMs = 100,
     stableChecksRequired = 3,
+    handles,
   } = options;
 
   if (!syncServerStorageId || !rootDirectoryUrl) {
@@ -47,26 +49,58 @@ export async function waitForBidirectionalSync(
   const startTime = Date.now();
   let lastSeenHeads = new Map<string, string>();
   let stableCount = 0;
-
-  debug(`waitForBidirectionalSync: starting (timeout=${timeoutMs}ms, stableChecks=${stableChecksRequired})`);
-
-  while (Date.now() - startTime < timeoutMs) {
-    // Get current heads for all documents in the directory hierarchy
-    const currentHeads = await getAllDocumentHeads(repo, rootDirectoryUrl);
+  let pollCount = 0;
+  let dynamicTimeoutMs = timeoutMs;
+
+  debug(`waitForBidirectionalSync: starting (timeout=${timeoutMs}ms, stableChecks=${stableChecksRequired}${handles ? `, tracking ${handles.length} handles` : ', full tree scan'})`);
+
+  while (Date.now() - startTime < dynamicTimeoutMs) {
+    pollCount++;
+    // Get current heads: use provided handles if available, otherwise full tree scan
+    const currentHeads = handles
+      ? getHandleHeads(handles)
+      : await getAllDocumentHeads(repo, rootDirectoryUrl);
+
+    // Scale timeout proportionally to tree size after first scan
+    if (pollCount === 1) {
+      dynamicTimeoutMs = Math.max(timeoutMs, 5000 + currentHeads.size * 50);
+      if (dynamicTimeoutMs !== timeoutMs) {
+        debug(`waitForBidirectionalSync: scaled timeout to ${dynamicTimeoutMs}ms for ${currentHeads.size} docs`);
+      }
+    }
 
     // Check if heads are stable (no changes since last check)
     const isStable = headsMapEqual(lastSeenHeads, currentHeads);
 
     if (isStable) {
       stableCount++;
-      debug(`waitForBidirectionalSync: stable check ${stableCount}/${stableChecksRequired} (${currentHeads.size} docs)`);
+      debug(`waitForBidirectionalSync: stable check ${stableCount}/${stableChecksRequired} (${currentHeads.size} docs, poll #${pollCount})`);
       if (stableCount >= stableChecksRequired) {
-        debug(`waitForBidirectionalSync: converged in ${Date.now() - startTime}ms`);
+        const elapsed = Date.now() - startTime;
+        debug(`waitForBidirectionalSync: converged in ${elapsed}ms after ${pollCount} polls (${currentHeads.size} docs)`);
+        out.taskLine(`Bidirectional sync converged (${currentHeads.size} docs, ${elapsed}ms)`);
         return; // Converged!
       }
     } else {
+      // Find which docs changed
+      if (lastSeenHeads.size > 0) {
+        const changedDocs: string[] = [];
+        for (const [url, heads] of currentHeads) {
+          if (lastSeenHeads.get(url) !== heads) {
+            changedDocs.push(url.slice(0, 20) + "...");
+          }
+        }
+        const newDocs = currentHeads.size - lastSeenHeads.size;
+        if (newDocs > 0) {
+          debug(`waitForBidirectionalSync: ${newDocs} new docs discovered, ${changedDocs.length} docs changed heads (poll #${pollCount})`);
+        } else if (changedDocs.length > 0) {
+          debug(`waitForBidirectionalSync: ${changedDocs.length} docs changed heads: ${changedDocs.slice(0, 5).join(", ")}${changedDocs.length > 5 ? ` ...and ${changedDocs.length - 5} more` : ""} (poll #${pollCount})`);
+        }
+      } else {
+        debug(`waitForBidirectionalSync: initial scan found ${currentHeads.size} docs (poll #${pollCount})`);
+      }
       if (stableCount > 0) {
-        debug(`waitForBidirectionalSync: heads changed, resetting stable count`);
+        debug(`waitForBidirectionalSync: heads changed after ${stableCount} stable checks, resetting`);
       }
       stableCount = 0;
       lastSeenHeads = currentHeads;
@@ -77,8 +111,23 @@ export async function waitForBidirectionalSync(
 
   // Timeout - but don't throw, just log a warning
   // The sync may still work, we just couldn't confirm stability
-  debug(`waitForBidirectionalSync: timed out after ${timeoutMs}ms`);
-  out.taskLine(`Sync stability check timed out after ${timeoutMs}ms`, true);
+  const elapsed = Date.now() - startTime;
+  debug(`waitForBidirectionalSync: timed out after ${elapsed}ms (${pollCount} polls, ${lastSeenHeads.size} docs tracked, reached ${stableCount}/${stableChecksRequired} stable checks)`);
+  out.taskLine(`Bidirectional sync timed out after ${(elapsed / 1000).toFixed(1)}s - document heads were still changing after ${pollCount} checks across ${lastSeenHeads.size} docs (reached ${stableCount}/${stableChecksRequired} stability checks). This may mean another peer is actively editing, or the sync server is slow to relay changes. The sync will continue but some remote changes may not be reflected yet.`, true);
+}
+
+/**
+ * Get heads from a pre-collected set of handles (cheap, synchronous reads).
+ * Used for post-push stabilization where we already know which documents changed.
+ */
+function getHandleHeads(
+  handles: DocHandle<unknown>[],
+): Map<string, string> {
+  const heads = new Map<string, string>();
+  for (const handle of handles) {
+    heads.set(getPlainUrl(handle.url), JSON.stringify(handle.heads()));
+  }
+  return heads;
 }
 
 /**
@@ -117,8 +166,8 @@ async function collectHeadsRecursive(
       return;
     }
 
-    // Process all entries in the directory
-    for (const entry of doc.docs) {
+    // Process all entries in the directory concurrently
+    await Promise.all(doc.docs.map(async (entry: { type: string; url: AutomergeUrl; name: string }) => {
       if (entry.type === "folder") {
         // Recurse into subdirectory (entry.url may have stale heads)
         await collectHeadsRecursive(repo, entry.url, heads);
@@ -132,7 +181,7 @@ async function collectHeadsRecursive(
           // File document may not exist yet
         }
       }
-    }
+    }));
   } catch {
     // Directory may not exist yet
   }
@@ -260,12 +309,23 @@ export async function waitForSync(
     });
   });
 
+  const needSync = handlesToWaitOn.length - alreadySynced;
+  if (needSync > 0) {
+    debug(`waitForSync: ${alreadySynced} already synced, waiting for ${needSync} remaining`);
+    out.taskLine(`Uploading: ${alreadySynced}/${handlesToWaitOn.length} already synced, waiting for ${needSync} more`);
+  } else {
+    debug(`waitForSync: all ${handlesToWaitOn.length} already synced`);
+  }
+
   try {
     await Promise.all(promises);
-    debug(`waitForSync: all ${handlesToWaitOn.length} documents synced in ${Date.now() - startTime}ms (${alreadySynced} were already synced)`);
+    const elapsed = Date.now() - startTime;
+    debug(`waitForSync: all ${handlesToWaitOn.length} documents synced in ${elapsed}ms (${alreadySynced} were already synced)`);
+    out.taskLine(`All ${handlesToWaitOn.length} documents uploaded to server (${(elapsed / 1000).toFixed(1)}s)`);
   } catch (error) {
     const elapsed = Date.now() - startTime;
     debug(`waitForSync: failed after ${elapsed}ms: ${error}`);
+    out.taskLine(`Upload to server failed after ${(elapsed / 1000).toFixed(1)}s: ${error}`, true);
     throw error;
   }
 }